Generation of micropython image with frozen modules

Makefile

@@ -1,26 +1,84 @@
 
+export PATH := $(CURDIR)/esp-open-sdk/xtensa-lx106-elf/bin:$(PATH)
+export CPATH := $(CURDIR)/esp-open-sdk/sdk/include:$(CPATH)
+
+MP_PORT = micropython/ports/esp8266
+
+MOD_DIR = $(MP_PORT)/modules
+
+MP_UNIX = micropython/ports/unix/micropython
+
+SRC_MODULES = src/app_loader.py src/settings.py
+
+# MOD_TO_SRC_REL = $(shell realpath --relative-to $(MOD_DIR) src)
+
+LIB_SYMLINKS = $(patsubst src/%.py,$(MOD_DIR)/%.py,$(SRC_MODULES))
+# LIB_SYMLINK_TARGETS = $(patsubst %.py,$(MOD_DIR)/%.py,$(SRC_MODULES))
 
 build: micropython
 
+clean:
+	$(MAKE) -C $(MP_PORT) clean clean-frozen
+	rm .requirements.txt.installed
 
-micropython: check-and-reinit-micropython-submodules
-	$(MAKE) --directory=micropython/mpy-cross
-	$(MAKE) --directory=micropython/ports/esp8266
+
+deploy: micropython
+	( \
+	    [ -f python2-venv/bin/activate ] && source python2-venv/bin/activate; \
+	    $(MAKE) -C $(MP_PORT) deploy BAUD=460800; \
+	)
+
+erase: micropython
+	( \
+	    [ -f python2-venv/bin/activate ] && source python2-venv/bin/activate; \
+	    $(MAKE) -C $(MP_PORT) erase; \
+	)
+
+
+check-esp-sdk:
+	@echo -n ":: Checking ESP SDK: "
+	@if (which xtensa-lx106-elf-gcc 2> /dev/null) ; then \
+	    echo "success"; \
+	else \
+	    echo "error: SDK not found. Add the location of xtensa-lx106-elf-gcc to PATH or run setup_sdk.sh to install the SDK locally"; \
+	    false; \
+	fi
+
+
+.requirements.txt.installed: requirements.txt
+	cat requirements.txt | xargs -n1 $(MP_UNIX) -m upip install -p $(MOD_DIR)
+	touch $@
+
+$(MOD_DIR)/%.py: src/%.py
+	ln -s $(shell realpath --relative-to $(MOD_DIR) $<) $@
+
+
+frozen-libs: .requirements.txt.installed $(LIB_SYMLINKS)
+
+micropython: micropython/ports/esp8266/build/firmware-combined.bin
+
+micropython/ports/esp8266/build/firmware-combined.bin: check-and-reinit-micropython-submodules check-esp-sdk frozen-libs
+	$(MAKE) -C micropython/mpy-cross
+	( \
+	    [ -f python2-venv/bin/activate ] && source python2-venv/bin/activate; \
+	    $(MAKE) -C $(MP_PORT); \
+	)
 
 
 check-and-reinit-submodules:
-	@echo "Checking submodules"
+	@echo ":: Checking submodules"
 	@if git submodule status | egrep -q '^[-]|^[+]' ; then \
 	    echo "INFO: Need to reinitialize git submodules"; \
 	    git submodule update --init; \
 	fi
 
-
 check-and-reinit-micropython-submodules: check-and-reinit-submodules
-	@echo "Checking micropython submodules"
+	@echo -n ":: Checking micropython submodules: "
 	@if cd micropython; git submodule status | egrep -q '^[-]|^[+]' ; then \
-	    echo "INFO: Need to reinitialize micropython git submodules"; \
+	    echo "info: Need to reinitialize micropython git submodules"; \
 	    git submodule update --init; \
+	else \
+	    echo "success"; \
 	fi
 
-.PHONY: check-and-reinit-submodules check-and-reinit-micropython-submodules
+.PHONY: check-and-reinit-submodules check-and-reinit-micropython-submodules check-esp-sdk clean

requirements.txt

@@ -0,0 +1,3 @@
+micropython-uasyncio
+micropython-umqtt.robust
+micropython-umqtt.simple

setup_sdk.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+here=$(dirname $(readlink -f $0))
+
+cd $here
+
+function log {
+    LIGHT_CYAN='\033[1;36m'
+    NC='\033[0m'
+    echo -e "${LIGHT_CYAN}:: $@${NC}"
+}
+
+log "Preparing esp-open-sdk dependencies"
+
+git submodule sync --recursive
+git submodule update --init --recursive
+
+log "Updating crosstool-NG to newer version for compatibility with bash 5"
+
+(cd esp-open-sdk/crosstool-NG; git checkout a4286b8c)
+
+
+if python --version 2>&1 | grep " 3."
+then
+    log "Python 3 detected. Creating python 2 virtualenv."
+    virtualenv2 python2-venv
+    source python2-venv/bin/activate
+    pip install pyserial
+fi
+
+log "Building esp-open-sdk"
+
+echo 'CT_DEBUG_gdb=n' >> esp-open-sdk/crosstool-config-overrides
+cd esp-open-sdk
+
+make STANDALONE=y

src/app_loader.py

@@ -118,7 +118,7 @@ async def update_app(mqtt, name):
         version_sub.unsubscribe()
 
         if current_app != name or current_version != latest:
-            mqtt.log('installed: {} v{}, want {} v{}'.format(
+            mqtt.log('Current: {} v{}, want {} v{}'.format(
                 current_app, current_version, name, latest))
             try:
                 # if the wrong app is installed, we remove it and reboot to
@@ -134,6 +134,9 @@ async def update_app(mqtt, name):
             app_size = await sub.wait_for_value()
             sub.unsubscribe()
             mqtt.log('Installed {} v{}, {}b'.format(name, latest, app_size))
+
+            install_dependencies(mqtt)
+
             return True
     except Exception as e:
         mqtt.log('Update error:', e)
@@ -142,6 +145,37 @@ async def update_app(mqtt, name):
     return False
 
 
+def install_dependencies(mqtt):
+    import app
+    try:
+        deps = app.DEPENDENCIES
+        mqtt.log('Installing {} dependencies'.format(len(deps)))
+        for d in deps:
+            gc.collect()
+            if len(d) == 1:  # upip package
+                mqtt.log('  - upip package {}'.format(d))
+                import upip
+                upip.install(d)
+            elif len(d) == 2:  # file path, source URL
+                path, url = d
+                mqtt.log('  - dependency {} from {}'.format(path, url))
+                gc.collect()
+                import upip
+                f = upip.url_open(url)
+                buf = bytearray(128)
+                view = memoryview(buf)
+                with open(path, 'wb') as outfile:
+                    while True:
+                        n_read = f.readinto(buf)
+                        if n_read == 0:
+                            break
+                        outfile.write(view[:n_read])
+            gc.collect()
+    except Exception as e:
+        mqtt.log('Error installing dependencies: ', e)
+
+
+
 def deepsleep(duration_ms):
     utime.sleep_ms(200)
     # configure RTC.ALARM0 to be able to wake the device