Get hot reloading working again

README.md

@@ -34,6 +34,9 @@ $ SERVER=127.0.0.1 python mosfet.py
 `mosfet.py` defaults to that server, player name `mosfet1`, and the current
 AuthMe password. Override `SERVER`, `PORT`, `PLAYER_NAME`, or `GAME_PASSWORD`
 through environment variables as needed. Microsoft authentication is not used.
+Python edits under `mosfet/` are hot-reloaded in place; the player connection is
+kept alive. Edits to connection/bootstrap files are reported as deferred until
+the next manual start, rather than triggering an automatic reconnect.
 
 The 26.3 client now handles AuthMe, chat commands, chunk/world-state caching,
 modern block/item/entity data, entity tracking, read-only inventory/component

mosfet.py

@@ -1,9 +1,11 @@
 import importlib
 import json
+import queue
 import threading
 import time
 import traceback
 import sys
+from pathlib import Path
 
 import logging
 log = logging.getLogger('werkzeug')
@@ -17,7 +19,7 @@ CORS(app)
 
 from munch import Munch
 from watchdog.observers import Observer
-from watchdog.events import PatternMatchingEventHandler
+from watchdog.events import FileSystemEventHandler
 
 from mosfet import connect
 
@@ -46,6 +48,7 @@ g.afk = False
 g.health = 20
 g.food = 20
 g.sand_origin = None
+g.reload_lock = threading.RLock()
 
 @app.route('/api/global')
 def api_global():
@@ -63,62 +66,177 @@ def root():
     return app.send_static_file('index.html')
 
 def main():
-    """Connect with offline/AuthMe and enable only the migrated 26.3 features.
+    """Run the 26.3 client with in-place development hot reload.
 
-    Current gameplay support includes path following and a crop-only harvest /
+    Application modules are refreshed while the current socket and player
-    replant job. General inventory actions, other block interactions, and
+    session remain alive. Connection/bootstrap and packet-schema edits are
-    legacy jobs remain disabled until migrated.
+    reported as deferred because their listeners cannot safely be replaced
+    without reconnecting.
     """
-    reload_timeout = time.time()
+    g.running = True
-    reload_requested = False
+    changes = queue.Queue()
-
-    def reload_modules():
-        manager_module = sys.modules.get('mosfet.protocol.managers')
-        if manager_module is not None:
-            importlib.reload(manager_module)
-        bot_module = sys.modules.get('mosfet.bot')
-        if bot_module is not None:
-            importlib.reload(bot_module)
-        importlib.reload(connect)
-
-    def reload_bot(event):
-        nonlocal reload_timeout, reload_requested
-        if time.time() - reload_timeout > 2.0:
-            reload_timeout = time.time()
-            reload_requested = True
-            print('Reloading...')
-            g.running = False
-
     observer = Observer()
-    event_handler = PatternMatchingEventHandler(
+    source_root = Path(__file__).resolve().parent / 'mosfet'
-        patterns=['*.py'], ignore_patterns=['./mosfet.py'])
-    event_handler.on_any_event = reload_bot
-    observer.schedule(event_handler, 'mosfet', recursive=True)
-    observer.start()
 
+    def enqueue_change(event):
+        if event.is_directory:
+            return
+        path = getattr(event, 'dest_path', None) if event.event_type == 'moved' else event.src_path
+        if not path or Path(path).suffix != '.py' or '__pycache__' in path:
+            return
+        changes.put(str(Path(path).resolve()))
+
+    def module_name(path):
+        source = Path(path)
         try:
+            relative = source.relative_to(source_root)
+        except ValueError:
+            return None
+        parts = list(relative.with_suffix('').parts)
+        if parts and parts[-1] == '__init__':
+            parts.pop()
+        return '.'.join(('mosfet', *parts))
+
+    def reload_changed(paths):
+        paths = sorted(set(path for path in paths if Path(path).is_file()))
+        if not paths:
+            return
+
+        # Don't execute a half-written editor save. If any file is syntactically
+        # invalid, keep the currently running code untouched.
+        try:
+            for path in paths:
+                source = Path(path)
+                compile(source.read_text(encoding='utf-8'), str(source), 'exec')
+        except (OSError, SyntaxError, UnicodeError):
+            print('[hot reload] waiting for a valid save', flush=True)
+            return
+
+        deferred = []
+        modules = []
+        for path in paths:
+            name = module_name(path)
+            if not name:
+                continue
+            if name in ('mosfet.connect', 'mosfet') or name == 'mosfet.monkey_patch':
+                deferred.append(name)
+            else:
+                modules.append(name)
+        if deferred:
+            print('[hot reload] {} changed; connection/bootstrap changes are '
+                  'deferred until the next start (no disconnect performed)'.format(
+                      ', '.join(sorted(set(deferred)))), flush=True)
+        if not modules:
+            return
+        if not g.get('connected') or not g.get('commands'):
+            deadline = time.monotonic() + 15.0
+            while (g.running and time.monotonic() < deadline and
+                   (not g.get('connected') or not g.get('commands'))):
+                time.sleep(0.1)
+            if not g.get('connected') or not g.get('commands'):
+                print('[hot reload] runtime not ready; change left for next save',
+                      flush=True)
+                return
+
+        managers_changed = bool(
+            {'mosfet.protocol.managers', 'mosfet.protocol.types'} & set(modules))
+        core_modules = {
+            'mosfet.bot', 'mosfet.commands', 'mosfet.game', 'mosfet.job',
+            'mosfet.path', 'mosfet.print_help', 'mosfet.utils',
+            'mosfet.vector', 'mosfet.world', 'mosfet.info.blocks',
+            'mosfet.info.items', 'mosfet.info.mcdata', 'mosfet.info.mobs',
+        }
+        # bot.py reloads the legacy gameplay modules and job state classes as
+        # one coherent group. Reload protocol adapters first so rebinding sees
+        # their latest definitions.
+        direct_modules = [name for name in modules
+                          if name not in core_modules and
+                          not name.startswith('mosfet.jobs.')]
+        if managers_changed and 'mosfet.protocol.managers' not in direct_modules:
+            direct_modules.append('mosfet.protocol.managers')
+        priority = {
+            'mosfet.protocol.types': 0,
+            'mosfet.protocol.packets': 1,
+            'mosfet.protocol.managers': 2,
+            'mosfet.protocol.modern': 3,
+        }
+        direct_modules.sort(key=lambda name: (priority.get(name, 10), name))
+
+        connection = g.connection
+        packet_lock = getattr(connection, 'packet_listener_lock', None)
+        try:
+            if packet_lock is None:
+                with g.reload_lock:
+                    apply_reload(direct_modules, managers_changed)
+            else:
+                # Match packet dispatch's lock order (listener lock, then the
+                # runtime lock) so callbacks cannot race listener replacement.
+                with packet_lock:
+                    with g.reload_lock:
+                        apply_reload(direct_modules, managers_changed)
+        except Exception:
+            traceback.print_exc()
+            print('[hot reload] failed; connection left running', flush=True)
+
+    def apply_reload(direct_modules, managers_changed):
+        for name in direct_modules:
+            module = sys.modules.get(name)
+            if module is not None:
+                importlib.reload(module)
+        bot_module = sys.modules.get('mosfet.bot')
+        if bot_module is None:
+            bot_module = importlib.import_module('mosfet.bot')
+        bot_module = importlib.reload(bot_module)
+        bot_module.refresh_runtime(global_state,
+                                   reload_managers=managers_changed)
+        print('[hot reload] refreshed {} (session preserved)'.format(
+            ', '.join(direct_modules) if direct_modules else 'bot runtime'),
+            flush=True)
+
+    def reload_worker():
+        while True:
+            path = changes.get()
+            if path is None:
+                return
+            batch = {path}
+            stopping = False
+            # Coalesce the burst of modified/moved events from editor saves.
             while True:
                 try:
-                g.running = True
+                    candidate = changes.get(timeout=0.4)
+                except queue.Empty:
+                    break
+                if candidate is None:
+                    stopping = True
+                    break
+                batch.add(candidate)
+            time.sleep(0.15)
+            reload_changed(batch)
+            if stopping:
+                return
+
+    event_handler = FileSystemEventHandler()
+    event_handler.on_modified = enqueue_change
+    event_handler.on_created = enqueue_change
+    event_handler.on_moved = enqueue_change
+    observer.schedule(event_handler, str(source_root), recursive=True)
+    observer.start()
+    worker = threading.Thread(target=reload_worker,
+                              name='mosfet-hot-reloader', daemon=True)
+    worker.start()
+
+    try:
         connect.main(global_state)
-                if not reload_requested:
-                    break
-                reload_requested = False
-                reload_modules()
     except KeyboardInterrupt:
-                break
+        pass
     except BaseException:
-                g.running = True
         traceback.print_exc()
-                print('Locking...')
-                while g.running:
-                    time.sleep(1)
-                reload_requested = False
-                reload_modules()
     finally:
         g.running = False
+        changes.put(None)
         observer.stop()
         observer.join()
+        worker.join(timeout=2.0)
 
 def run_api():
     host = '0.0.0.0'

mosfet/bot.py

@@ -378,10 +378,11 @@ def bot(global_state):
 print('Bot module loaded.')
 
 
-def start_automation(global_state):
+def start_automation(global_state, initialize=True):
-    """Start migrated path-following and crop automation only."""
+    """Start or replace the movement worker without replacing the connection."""
     g = global_state
     g.protocol_version = 777
+    if initialize:
         g.time = 0.0
         g.path = []
         g.look_at = None
@@ -397,18 +398,25 @@ def start_automation(global_state):
         g.yaw = float(g.get('yaw', 0.0) or 0.0)
         g.pitch = float(g.get('pitch', 0.0) or 0.0)
         g.chunks.loading = False
-    # Readiness below is based on the bot's own chunk and the next two path
+        # Readiness below is based on nearby chunks, not unrelated view holes.
-    # steps, never on unrelated holes elsewhere in the view cache.
         g.movement_paused = False
         g.movement_status = 'idle'
         g.movement_error = None
-    # Keep the port constrained to crop harvesting; other legacy jobs and
+        # Keep the active port constrained to the migrated job subset.
-    # general inventory/window mutations are not exposed through this adapter.
         from mosfet.game import Game
         from mosfet.job import JobStates
         g.game = Game(g)
         g.job = JobStates(g)
 
+    old_stop = g.get('movement_stop_event')
+    old_worker = g.get('movement_thread')
+    if old_stop is not None:
+        old_stop.set()
+    if old_worker is not None and old_worker is not threading.current_thread():
+        old_worker.join(timeout=1.0)
+    stop_event = threading.Event()
+    g.movement_stop_event = stop_event
+
     def run():
         from mosfet import bot, utils
         loaded_since = None
@@ -419,8 +427,8 @@ def start_automation(global_state):
                 print('[movement] status={}'.format(status), flush=True)
                 g.movement_status = status
 
-        while (g.get('running', True) and g.get('connection') and
+        while (not stop_event.is_set() and g.get('running', True) and
-               g.connection.connected):
+               g.get('connection') and g.connection.connected):
             started = time.monotonic()
             now = time.monotonic()
             if not initial_world_ready:
@@ -443,6 +451,11 @@ def start_automation(global_state):
                 set_status('waiting_for_chunks')
             elif has_work and movement_ready:
                 try:
+                    reload_lock = g.get('reload_lock')
+                    if reload_lock is None:
+                        bot.tick(g)
+                    else:
+                        with reload_lock:
                             bot.tick(g)
                 except Exception:
                     traceback.print_exc()
@@ -462,3 +475,63 @@ def start_automation(global_state):
     g.movement_thread = worker
     worker.start()
     return worker
+
+
+def _remove_instance_listeners(connection, instance):
+    if connection is None or instance is None:
+        return
+    for name in ('packet_listeners', 'early_packet_listeners',
+                 'outgoing_packet_listeners',
+                 'early_outgoing_packet_listeners'):
+        listeners = getattr(connection, name, [])
+        listeners[:] = [listener for listener in listeners
+                        if getattr(getattr(listener, 'callback', None),
+                                   '__self__', None) is not instance]
+
+
+def refresh_runtime(global_state, reload_managers=False):
+    """Rebind reloadable runtime adapters without touching the connection."""
+    g = global_state
+    connection = g.get('connection')
+
+    _remove_instance_listeners(connection, g.get('chat'))
+    _remove_instance_listeners(connection, g.get('game'))
+    if reload_managers:
+        _remove_instance_listeners(connection, g.get('entities'))
+        _remove_instance_listeners(connection, g.get('inventory'))
+
+    if reload_managers:
+        from mosfet.protocol import managers
+        if connection and g.get('chunks'):
+            current = g.chunks
+            replacement = managers.WorldState(current.min_y)
+            replacement.__dict__.update(current.__dict__)
+            g.chunks = replacement
+            g.world_state = replacement
+            connection.world_state = replacement
+        from mosfet.info import blocks as block_data
+        from mosfet.info import items as item_data
+        g.mcdata = DataManager()
+        g.entities = managers.EntityManager(g)
+        g.inventory = managers.InventoryManager(g)
+        # Keep data references in the reloaded legacy Game adapter current.
+        _ = (block_data, item_data)
+
+    from mosfet.protocol.modern import ModernChatManager
+    g.chat = ModernChatManager(g)
+    g.world = world.World(g)
+    g.game = game.Game(g)
+
+    state_name = getattr(g.get('job'), 'state', None)
+    state_name = getattr(state_name, '__name__', None)
+    g.job = job.JobStates(g)
+    if state_name and state_name != 'idle':
+        replacement_state = getattr(g.job, state_name, None)
+        if replacement_state is not None:
+            g.job.state = replacement_state
+    g.commands = commands.Commands(g)
+    if not reload_managers:
+        g.mcdata = DataManager()
+    start_automation(g, initialize=False)
+    print('[hot reload] runtime adapters refreshed; connection kept alive',
+          flush=True)

mosfet/protocol/modern.py

@@ -1,5 +1,6 @@
 """Chat adapter for the Minecraft 26.3 play protocol."""
 import time
+from contextlib import nullcontext
 
 from minecraft.networking.modern import (
     ChatCommand, ClientboundPlayerChatPacket, ServerboundChatPacket, SystemChat,
@@ -21,10 +22,12 @@ class ModernChatManager:
         text = '{}: {}'.format(sender, packet.message)
         print('[chat] {}: {}'.format(sender, packet.message), flush=True)
         if self.handler:
+            with self.g.get('reload_lock') or nullcontext():
                 self.handler(('CHAT', sender, text))
 
     def handle_system_chat(self, packet, message):
         if self.handler:
+            with self.g.get('reload_lock') or nullcontext():
                 self.handler(('SYSTEM', None, message))
 
     def send(self, message):