separate .json for boards, use slurpfile

available since jq version 1.5 that was released on Sat Aug 15 2015
esp8266 · mcspr · Feb 8, 2022 · Feb 14, 2022 · Apr 25, 2022 · Apr 30, 2022
commit 123057fdf9cfb5a8eb04403d4c6436c148307728
diff --git a/package/build_boards_manager_package.sh b/package/build_boards_manager_package.sh
@@ -146,8 +146,12 @@ if [ ! -z "${DOC_URL}" ]; then
         .packages[0].platforms[0].help.online = \"${DOC_URL}\""
 fi
 
-cat ${srcdir}/package/package_esp8266com_index.template.json | \
-    jq "${jq_arg}" > package_esp8266com_index.json
+jq_arg="${jq_arg} |\
+    .packages[0].platforms[0].boards = "'$'"boards"
+
+jq --slurpfile boards ${srcdir}/package/package_esp8266com_index.boards.json \
+    "${jq_arg}" ${srcdir}/package/package_esp8266com_index.template.json \
+    > package_esp8266com_index.json
 
 # Use Github API token, if available
 curl_gh_token_arg=()

diff --git a/package/package_esp8266com_index.boards.json b/package/package_esp8266com_index.boards.json
@@ -0,0 +1,113 @@
+[
+    {
+        "name": "Generic ESP8266 Module"
+    },
+    {
+        "name": "Generic ESP8285 Module"
+    },
+    {
+        "name": "Lifely Agrumino Lemon v4"
+    },
+    {
+        "name": "ESPDuino (ESP-13 Module)"
+    },
+    {
+        "name": "Adafruit Feather HUZZAH ESP8266"
+    },
+    {
+        "name": "WiFi Kit 8"
+    },
+    {
+        "name": "Invent One"
+    },
+    {
+        "name": "XinaBox CW01"
+    },
+    {
+        "name": "ESPresso Lite 1.0"
+    },
+    {
+        "name": "ESPresso Lite 2.0"
+    },
+    {
+        "name": "Phoenix 1.0"
+    },
+    {
+        "name": "Phoenix 2.0"
+    },
+    {
+        "name": "NodeMCU 0.9 (ESP-12 Module)"
+    },
+    {
+        "name": "NodeMCU 1.0 (ESP-12E Module)"
+    },
+    {
+        "name": "Olimex MOD-WIFI-ESP8266(-DEV)"
+    },
+    {
+        "name": "SparkFun ESP8266 Thing"
+    },
+    {
+        "name": "SparkFun ESP8266 Thing Dev"
+    },
+    {
+        "name": "SparkFun Blynk Board"
+    },
+    {
+        "name": "SweetPea ESP-210"
+    },
+    {
+        "name": "LOLIN(WEMOS) D1 R2 & mini"
+    },
+    {
+        "name": "LOLIN(WEMOS) D1 mini (clone)"
+    },
+    {
+        "name": "LOLIN(WEMOS) D1 mini Pro"
+    },
+    {
+        "name": "LOLIN(WEMOS) D1 mini Lite"
+    },
+    {
+        "name": "LOLIN(WeMos) D1 R1"
+    },
+    {
+        "name": "ESPino (ESP-12 Module)"
+    },
+    {
+        "name": "ThaiEasyElec's ESPino"
+    },
+    {
+        "name": "WifInfo"
+    },
+    {
+        "name": "Arduino"
+    },
+    {
+        "name": "4D Systems gen4 IoD Range"
+    },
+    {
+        "name": "Digistump Oak"
+    },
+    {
+        "name": "WiFiduino"
+    },
+    {
+        "name": "Amperka WiFi Slot"
+    },
+    {
+        "name": "Seeed Wio Link"
+    },
+    {
+        "name": "ESPectro Core"
+    },
+    {
+        "name": "Schirmilabs Eduino WiFi"
+    },
+    {
+        "name": "ITEAD Sonoff"
+    },
+    {
+        "name": "DOIT ESP-Mx DevKit (ESP8285)"
+    }
+]
diff --git a/package/package_esp8266com_index.template.json b/package/package_esp8266com_index.template.json
@@ -16,119 +16,7 @@
                "version": "",
                "architecture": "esp8266",
                "archiveFileName": "",
-               "boards": [
-                  {
-                     "name": "Generic ESP8266 Module"
-                  },
-                  {
-                     "name": "Generic ESP8285 Module"
-                  },
-                  {
-                     "name": "Lifely Agrumino Lemon v4"
-                  },
-                  {
-                     "name": "ESPDuino (ESP-13 Module)"
-                  },
-                  {
-                     "name": "Adafruit Feather HUZZAH ESP8266"
-                  },
-                  {
-                     "name": "WiFi Kit 8"
-                  },
-                  {
-                     "name": "Invent One"
-                  },
-                  {
-                     "name": "XinaBox CW01"
-                  },
-                  {
-                     "name": "ESPresso Lite 1.0"
-                  },
-                  {
-                     "name": "ESPresso Lite 2.0"
-                  },
-                  {
-                     "name": "Phoenix 1.0"
-                  },
-                  {
-                     "name": "Phoenix 2.0"
-                  },
-                  {
-                     "name": "NodeMCU 0.9 (ESP-12 Module)"
-                  },
-                  {
-                     "name": "NodeMCU 1.0 (ESP-12E Module)"
-                  },
-                  {
-                     "name": "Olimex MOD-WIFI-ESP8266(-DEV)"
-                  },
-                  {
-                     "name": "SparkFun ESP8266 Thing"
-                  },
-                  {
-                     "name": "SparkFun ESP8266 Thing Dev"
-                  },
-                  {
-                     "name": "SparkFun Blynk Board"
-                  },
-                  {
-                     "name": "SweetPea ESP-210"
-                  },
-                  {
-                     "name": "LOLIN(WEMOS) D1 R2 & mini"
-                  },
-                  {
-                     "name": "LOLIN(WEMOS) D1 mini (clone)"
-                  },
-                  {
-                     "name": "LOLIN(WEMOS) D1 mini Pro"
-                  },
-                  {
-                     "name": "LOLIN(WEMOS) D1 mini Lite"
-                  },
-                  {
-                     "name": "LOLIN(WeMos) D1 R1"
-                  },
-                  {
-                     "name": "ESPino (ESP-12 Module)"
-                  },
-                  {
-                     "name": "ThaiEasyElec's ESPino"
-                  },
-                  {
-                     "name": "WifInfo"
-                  },
-                  {
-                     "name": "Arduino"
-                  },
-                  {
-                     "name": "4D Systems gen4 IoD Range"
-                  },
-                  {
-                     "name": "Digistump Oak"
-                  },
-                  {
-                     "name": "WiFiduino"
-                  },
-                  {
-                     "name": "Amperka WiFi Slot"
-                  },
-                  {
-                     "name": "Seeed Wio Link"
-                  },
-                  {
-                     "name": "ESPectro Core"
-                  },
-                  {
-                     "name": "Schirmilabs Eduino WiFi"
-                  },
-                  {
-                     "name": "ITEAD Sonoff"
-                  },
-                  {
-                     "name": "DOIT ESP-Mx DevKit (ESP8285)"
-                  }
-               ],
+               "boards": [],
                "toolsDependencies": [
                   {
                      "packager": "esp8266",
@@ -380,4 +268,4 @@
          ]
       }
    ]
-}
+}
diff --git a/tools/boards.txt.py b/tools/boards.txt.py
@@ -1984,7 +1984,10 @@ def size(value):
     if not fs:
         fs = Filesystem(Region("", eeprom.start, eeprom.start), 0, 0)
 
-    with context(os.path.join(output, ld)):
+    if output and not os.path.isdir(output):
+        raise TypeError('.ld output must be a directory')
+
+    with context(os.path.join(output, ld) if output else None):
         print(f'/* Flash Split for {size(layout)} chips */')
         print(f'/* sketch @{address(sketch)} (~{size(sketch)}) ({sketch.size}B) */')
         if empty:
@@ -2389,22 +2392,15 @@ def boardnames (boards):
 ################################################################
 
 def package_generate (output, boards):
-    with open(output, "r", encoding="utf-8") as f:
-        data = json.load(f, object_pairs_hook=collections.OrderedDict)
-
-    target = None
-    for package in data["packages"]:
-        for platform in package["platforms"]:
-            if platform["name"] == "esp8266":
-                target = platform["boards"]
-
-    if not target:
-        raise ValueError('no "boards" field found for the platform')
+    context = contextlib.nullcontext
+    if output:
+        context = OpenWithBackupFile
 
-    with OpenWithBackupFile(output):
+    with context(output):
         target = [{"name": board["name"]} for board in boards.values()]
         print(json.dumps(data, indent=3, separators=(",", ": ")))
 
+
 ################################################################
 
 def doc_generate (output, boards):
@@ -2444,23 +2440,24 @@ def parse_cmdline ():
     filters.add_argument("--exclude", nargs="?", help="resulting BOARDSFILE will *not* include boards listed in the EXCLUDE file")
 
     generators = [
-        ["boards", "boardsfile", "boards.txt", "boards.txt"],
-        ["ld", "lddir", "tools/sdk/ld", ".ld scripts"],
-        ["flashmap", "flashmapfile", "cores/esp8266/FlashMap.h", "FlashMap header"],
-        ["package", "packagefile", "package/package_esp8266com_index.template.json", "package template .json"],
-        ["doc", "docfile", "doc/boards.rst", "boards documentation"],
+        ["boards", "boards-file", "boards.txt", "boards.txt"],
+        ["ld", "ld-dir", "tools/sdk/ld", ".ld scripts"],
+        ["flashmap", "flashmap-file", "cores/esp8266/FlashMap.h", "FlashMap header"],
+        ["package", "package-file", "package/package_esp8266com_index.boards.json", "IDE package index boards list (.json)"],
+        ["doc", "doc-file", "doc/boards.rst", "Boards documentation (.rst)"],
     ]
 
     for name, output, default, title in generators:
         sub = parser.add_argument_group(title=title)
         sub.add_argument(f'--{name}', dest="generators", action="append_const", const=name)
         sub.add_argument(f'--{output}', default=default)
 
-    parser.add_argument("--all", dest="generators", action="store_const", const=[
+    generate_all = parser.add_argument_group("Use all available generators")
+    generate_all.add_argument("--all", dest="generators", action="store_const", const=[
         name for name, _, _, _ in generators
-    ], help="Select all available generators")
+    ])
 
-    parser.add_argument("--generate", action="store_true", help="Generate and write to selected files")
+    parser.add_argument("--output", action="store", choices=("file", "stdout"), default="stdout", help="Generators output to file (or directory) or stdout (default)")
 
     return parser.parse_args()
 
@@ -2482,11 +2479,14 @@ def main ():
         boardnames(boards)
         return
 
+    def maybe_output(file):
+        return file if args.output == "file" else None
+
     generators = set(name for name in args.generators or [])
 
     if "boards" in generators:
         all_boards_generate(
-            args.boardsfile if args.generate else None,
+            maybe_output(args.boards_file),
             boards,
             macros=prepare_macros(MACROS, all_flash_maps(), args.led),
             extra_header=[
@@ -2499,24 +2499,16 @@ def main ():
             ])
 
     if "ld" in generators:
-        all_ldscript_generate(
-            args.lddir if args.generate else None,
-            all_flash_maps())
+        all_ldscript_generate(maybe_output(args.ld_dir), all_flash_maps())
 
     if "flashmap" in generators:
-        flashmap_generate(
-            args.flashmapfile if args.generate else None,
-            all_flash_maps())
+        flashmap_generate(maybe_output(args.flashmap_file), all_flash_maps())
 
     if "package" in generators:
-        package_generate(
-            args.packagefile if args.generate else None,
-            boards)
+        package_generate(maybe_output(args.package_file), boards)
 
     if "doc" in generators:
-        doc_generate(
-            args.docfile if args.generate else None,
-            boards)
+        doc_generate(maybe_output(args.doc_file), boards)
 
 
 if __name__ == "__main__":