Automate via workflows. Add auto-archiving for downgrade. (#581)

src/autodl/github.ts

@@ -0,0 +1,95 @@
+import {getJson, getLatestImage, readCacheJson, writeCacheJson} from '../common.js';
+import {processFirmwareImage} from '../process_firmware_image.js';
+
+type ReleaseAssetJson = {
+    url: string;
+    id: number;
+    node_id: string;
+    name: string;
+    label: null;
+    uploader: Record<string, unknown>;
+    content_type: string;
+    state: string;
+    size: number;
+    download_count: number;
+    created_at: string;
+    updated_at: string;
+    browser_download_url: string;
+};
+type ReleaseJson = {
+    url: string;
+    assets_url: string;
+    upload_url: string;
+    html_url: string;
+    id: number;
+    author: Record<string, unknown>;
+    node_id: string;
+    tag_name: string;
+    target_commitish: string;
+    name: string;
+    draft: false;
+    prerelease: false;
+    created_at: string;
+    published_at: string;
+    assets: ReleaseAssetJson[];
+    tarball_url: string;
+    zipball_url: string;
+    body: string;
+    reactions: Record<string, unknown>;
+};
+type ReleasesJson = ReleaseJson[];
+type AssetFindPredicate = (value: ReleaseAssetJson, index: number, obj: ReleaseAssetJson[]) => unknown;
+
+function sortByPublishedAt(a: ReleaseJson, b: ReleaseJson): number {
+    return a.published_at < b.published_at ? -1 : a.published_at > b.published_at ? 1 : 0;
+}
+
+function isDifferent(newData: ReleaseAssetJson, cachedData?: ReleaseAssetJson): boolean {
+    return Boolean(process.env.IGNORE_CACHE) || !cachedData || cachedData.updated_at !== newData.updated_at;
+}
+
+export async function writeCache(manufacturer: string, releasesUrl: string): Promise<void> {
+    const releases = await getJson<ReleasesJson>(manufacturer, releasesUrl);
+
+    if (releases?.length) {
+        writeCacheJson(manufacturer, releases);
+    }
+}
+
+export async function download(manufacturer: string, releasesUrl: string, assetFinders: AssetFindPredicate[]): Promise<void> {
+    const logPrefix = `[${manufacturer}]`;
+    const releases = await getJson<ReleasesJson>(manufacturer, releasesUrl);
+
+    if (releases?.length) {
+        const release = getLatestImage(releases, sortByPublishedAt);
+
+        if (release) {
+            const cachedData = readCacheJson<ReleasesJson | undefined>(manufacturer);
+            const cached = cachedData?.length ? getLatestImage(cachedData, sortByPublishedAt) : undefined;
+
+            for (const assetFinder of assetFinders) {
+                const asset = release.assets.find(assetFinder);
+
+                if (asset) {
+                    const cachedAsset = cached?.assets.find(assetFinder);
+
+                    if (!isDifferent(asset, cachedAsset)) {
+                        console.log(`[${manufacturer}:${asset.name}] No change from last run.`);
+                        continue;
+                    }
+
+                    await processFirmwareImage(manufacturer, asset.name, asset.browser_download_url, {
+                        manufacturerName: [manufacturer],
+                        releaseNotes: release.html_url,
+                    });
+                } else {
+                    console.error(`${logPrefix} No image found.`);
+                }
+            }
+        } else {
+            console.error(`${logPrefix} No release found.`);
+        }
+
+        writeCacheJson(manufacturer, releases);
+    }
+}

src/autodl/gmmts.ts

@@ -0,0 +1,76 @@
+import {getJson, readCacheJson, writeCacheJson} from '../common.js';
+import {processFirmwareImage} from '../process_firmware_image.js';
+
+type ImagesJsonBuildPart = {
+    path: string; // .bin
+    offset: number;
+    type?: 'app' | 'storage';
+    ota?: string; // .ota
+};
+type ImagesJsonBuild = {
+    chipFamily: string;
+    target: string;
+    parts: ImagesJsonBuildPart[];
+};
+type ImagesJson = {
+    name: string;
+    version: string;
+    home_assistant_domain: string;
+    funding_url: string;
+    new_install_prompt_erase: boolean;
+    builds: ImagesJsonBuild[];
+};
+
+const NAME = 'Gmmts';
+// const LOG_PREFIX = `[${NAME}]`;
+const BASE_URL = 'https://update.gammatroniques.fr/';
+const MANIFEST_URL_PATH = `/manifest.json`;
+const MODEL_IDS = ['ticmeter'];
+
+function isDifferent(newData: ImagesJson, cachedData?: ImagesJson): boolean {
+    return Boolean(process.env.IGNORE_CACHE) || !cachedData || cachedData.version !== newData.version;
+}
+
+export async function writeCache(): Promise<void> {
+    for (const modelId of MODEL_IDS) {
+        const url = `${BASE_URL}${modelId}${MANIFEST_URL_PATH}`;
+        const page = await getJson<ImagesJson>(NAME, url);
+
+        if (page?.builds?.length) {
+            writeCacheJson(`${NAME}_${modelId}`, page);
+        }
+    }
+}
+
+export async function download(): Promise<void> {
+    for (const modelId of MODEL_IDS) {
+        const logPrefix = `[${NAME}:${modelId}]`;
+        const url = `${BASE_URL}${modelId}${MANIFEST_URL_PATH}`;
+        const page = await getJson<ImagesJson>(NAME, url);
+
+        if (!page?.builds?.length) {
+            console.error(`${logPrefix} No image data.`);
+            continue;
+        }
+
+        const cacheFileName = `${NAME}_${modelId}`;
+
+        if (!isDifferent(page, readCacheJson(cacheFileName))) {
+            console.log(`${logPrefix} No change from last run.`);
+            continue;
+        }
+
+        writeCacheJson(cacheFileName, page);
+
+        const appUrl: ImagesJsonBuildPart | undefined = page.builds[0].parts.find((part) => part.type === 'app');
+
+        if (!appUrl || !appUrl.ota) {
+            console.error(`${logPrefix} No image found.`);
+            continue;
+        }
+
+        const firmwareFileName = appUrl.ota.split('/').pop()!;
+
+        await processFirmwareImage(NAME, firmwareFileName, appUrl.ota, {manufacturerName: [NAME]});
+    }
+}

src/autodl/ikea.ts

@@ -0,0 +1,84 @@
+import {getJson, readCacheJson, writeCacheJson} from '../common.js';
+import {processFirmwareImage} from '../process_firmware_image.js';
+
+type GatewayImageJson = {
+    fw_binary_url: string;
+    fw_filesize: number;
+    fw_hotfix_version: number;
+    fw_major_version: number;
+    fw_minor_version: number;
+    fw_req_hotfix_version: number;
+    fw_req_major_version: number;
+    fw_req_minor_version: number;
+    fw_type: 0;
+    fw_update_prio: number;
+    fw_weblink_relnote: string;
+};
+type DeviceImageJson = {
+    fw_binary_url: string;
+    fw_file_version_LSB: number;
+    fw_file_version_MSB: number;
+    fw_filesize: number;
+    fw_image_type: number;
+    fw_manufacturer_id: number;
+    fw_type: 2;
+};
+type ImagesJson = (GatewayImageJson | DeviceImageJson)[];
+
+const NAME = 'IKEA';
+const LOG_PREFIX = `[${NAME}]`;
+const PRODUCTION_FIRMWARE_URL = 'http://fw.ota.homesmart.ikea.net/feed/version_info.json';
+// const TEST_FIRMWARE_URL = 'http://fw.test.ota.homesmart.ikea.net/feed/version_info.json';
+export const RELEASE_NOTES_URL = 'https://ww8.ikea.com/ikeahomesmart/releasenotes/releasenotes.html';
+
+function findInCache(image: DeviceImageJson, cachedData?: ImagesJson): DeviceImageJson | undefined {
+    // `fw_type` compare ensures always `DeviceImagesJson`
+    return cachedData?.find(
+        (d) => d.fw_type == image.fw_type && d.fw_image_type == image.fw_image_type && d.fw_manufacturer_id == image.fw_manufacturer_id,
+    ) as DeviceImageJson | undefined;
+}
+
+function isDifferent(newData: DeviceImageJson, cachedData?: DeviceImageJson): boolean {
+    return (
+        Boolean(process.env.IGNORE_CACHE) ||
+        !cachedData ||
+        cachedData.fw_file_version_LSB !== newData.fw_file_version_LSB ||
+        cachedData.fw_file_version_MSB !== newData.fw_file_version_MSB
+    );
+}
+
+export async function writeCache(): Promise<void> {
+    const images = await getJson<ImagesJson>(NAME, PRODUCTION_FIRMWARE_URL);
+
+    if (images?.length) {
+        writeCacheJson(NAME, images);
+    }
+}
+
+export async function download(): Promise<void> {
+    const images = await getJson<ImagesJson>(NAME, PRODUCTION_FIRMWARE_URL);
+
+    if (images?.length) {
+        const cachedData = readCacheJson<ImagesJson | undefined>(NAME);
+
+        for (const image of images) {
+            if (image.fw_type !== 2) {
+                // ignore gateway firmware
+                continue;
+            }
+
+            const firmwareFileName = image.fw_binary_url.split('/').pop()!;
+
+            if (!isDifferent(image, findInCache(image, cachedData))) {
+                console.log(`[${NAME}:${firmwareFileName}] No change from last run.`);
+                continue;
+            }
+
+            await processFirmwareImage(NAME, firmwareFileName, image.fw_binary_url, {manufacturerName: [NAME], releaseNotes: RELEASE_NOTES_URL});
+        }
+
+        writeCacheJson(NAME, images);
+    } else {
+        console.error(`${LOG_PREFIX} No image data.`);
+    }
+}

src/autodl/ikea_new.ts

@@ -0,0 +1,75 @@
+import {getJson, readCacheJson, writeCacheJson} from '../common.js';
+import {processFirmwareImage} from '../process_firmware_image.js';
+import {RELEASE_NOTES_URL} from './ikea.js';
+
+type GatewayImageJson = {
+    fw_type: 3;
+    fw_sha3_256: string;
+    fw_binary_url: string;
+    fw_update_prio: number;
+    fw_filesize: number;
+    fw_minor_version: number;
+    fw_major_version: number;
+    fw_hotfix_version: number;
+    fw_binary_checksum: string;
+};
+type DeviceImageJson = {
+    fw_image_type: number;
+    fw_type: 2;
+    fw_sha3_256: string;
+    fw_binary_url: string;
+};
+
+type ImagesJson = (GatewayImageJson | DeviceImageJson)[];
+
+// same name as `ikea.ts` to keep everything in same folder
+const NAME = 'IKEA';
+const CACHE_FILENAME = `${NAME}_new`;
+const LOG_PREFIX = `[${NAME}_new]`;
+// requires cacerts/ikea_new.pem
+const FIRMWARE_URL = 'https://fw.ota.homesmart.ikea.com/check/update/prod';
+
+function findInCache(image: DeviceImageJson, cachedData?: ImagesJson): DeviceImageJson | undefined {
+    // `fw_type` compare ensures always `DeviceImagesJson`
+    return cachedData?.find((d) => d.fw_type == image.fw_type && d.fw_image_type == image.fw_image_type) as DeviceImageJson | undefined;
+}
+
+function isDifferent(newData: DeviceImageJson, cachedData?: DeviceImageJson): boolean {
+    return Boolean(process.env.IGNORE_CACHE) || !cachedData || cachedData.fw_sha3_256 !== newData.fw_sha3_256;
+}
+
+export async function writeCache(): Promise<void> {
+    const images = await getJson<ImagesJson>(NAME, FIRMWARE_URL);
+
+    if (images?.length) {
+        writeCacheJson(CACHE_FILENAME, images);
+    }
+}
+
+export async function download(): Promise<void> {
+    const images = await getJson<ImagesJson>(NAME, FIRMWARE_URL);
+
+    if (images?.length) {
+        const cachedData = readCacheJson<ImagesJson | undefined>(CACHE_FILENAME);
+
+        for (const image of images) {
+            if (image.fw_type !== 2) {
+                // ignore gateway firmware
+                continue;
+            }
+
+            const firmwareFileName = image.fw_binary_url.split('/').pop()!;
+
+            if (!isDifferent(image, findInCache(image, cachedData))) {
+                console.log(`[${NAME}:${firmwareFileName}] No change from last run.`);
+                continue;
+            }
+
+            await processFirmwareImage(NAME, firmwareFileName, image.fw_binary_url, {manufacturerName: [NAME], releaseNotes: RELEASE_NOTES_URL});
+        }
+
+        writeCacheJson(CACHE_FILENAME, images);
+    } else {
+        console.error(`${LOG_PREFIX} No image data.`);
+    }
+}

src/autodl/inovelli.ts

@@ -0,0 +1,72 @@
+import {getJson, getLatestImage, readCacheJson, writeCacheJson} from '../common.js';
+import {processFirmwareImage} from '../process_firmware_image.js';
+
+type DeviceImageJson = {
+    version: string;
+    channel: 'beta' | 'production';
+    firmware: string;
+    manufacturer_id: number;
+    image_type: number;
+};
+type ModelsJson = {
+    [k: string]: DeviceImageJson[];
+};
+
+const NAME = 'Inovelli';
+const LOG_PREFIX = `[${NAME}]`;
+const FIRMWARE_URL = 'https://files.inovelli.com/firmware/firmware.json';
+
+function sortByVersion(a: DeviceImageJson, b: DeviceImageJson): number {
+    const aRadix = a.version.match(/[a-fA-F]/) ? 16 : 10;
+    const bRadix = b.version.match(/[a-fA-F]/) ? 16 : 10;
+    const aVersion = parseInt(a.version, aRadix);
+    const bVersion = parseInt(b.version, bRadix);
+
+    return aVersion < bVersion ? -1 : aVersion > bVersion ? 1 : 0;
+}
+
+function isDifferent(newData: DeviceImageJson, cachedData?: DeviceImageJson): boolean {
+    return Boolean(process.env.IGNORE_CACHE) || !cachedData || cachedData.version !== newData.version;
+}
+
+export async function writeCache(): Promise<void> {
+    const models = await getJson<ModelsJson>(NAME, FIRMWARE_URL);
+
+    if (models) {
+        writeCacheJson(NAME, models);
+    }
+}
+
+export async function download(): Promise<void> {
+    const models = await getJson<ModelsJson>(NAME, FIRMWARE_URL);
+
+    if (models) {
+        const cachedData = readCacheJson<ModelsJson | undefined>(NAME);
+
+        for (const model in models) {
+            if (model == '') {
+                // ignore empty key (bug)
+                continue;
+            }
+
+            const image = getLatestImage(models[model], sortByVersion);
+
+            if (!image) {
+                continue;
+            }
+
+            const firmwareFileName = image.firmware.split('/').pop()!;
+
+            if (cachedData && !isDifferent(image, getLatestImage(cachedData[model], sortByVersion))) {
+                console.log(`[${NAME}:${firmwareFileName}] No change from last run.`);
+                continue;
+            }
+
+            await processFirmwareImage(NAME, firmwareFileName, image.firmware, {manufacturerName: [NAME]});
+        }
+
+        writeCacheJson(NAME, models);
+    } else {
+        console.error(`${LOG_PREFIX} No image data.`);
+    }
+}

src/autodl/jethome.ts

@@ -0,0 +1,91 @@
+import {getJson, readCacheJson, writeCacheJson} from '../common.js';
+import {processFirmwareImage} from '../process_firmware_image.js';
+
+type ImageJson = {
+    vendor: string;
+    vendor_name: string;
+    device: string;
+    device_name: string;
+    platform: string;
+    platform_name: string;
+    latest_firmware: {
+        release: {
+            version: string;
+            date: string;
+            images: {
+                'zigbee.ota': {
+                    url: string;
+                    hash: string;
+                    filesize: number;
+                };
+                'zigbee.bin': {
+                    url: string;
+                    hash: string;
+                    filesize: number;
+                };
+            };
+            changelog: string;
+        };
+    };
+};
+
+const NAME = 'Jethome';
+const LOG_PREFIX = `[${NAME}]`;
+const BASE_URL = 'https://fw.jethome.ru';
+const DEVICE_URL = `${BASE_URL}/api/devices/`;
+
+const MODEL_IDS = ['WS7'];
+
+function getCacheFileName(modelId: string): string {
+    return `${NAME}_${modelId}`;
+}
+
+function isDifferent(newData: ImageJson, cachedData?: ImageJson): boolean {
+    return Boolean(process.env.IGNORE_CACHE) || !cachedData || cachedData.latest_firmware.release.version !== newData.latest_firmware.release.version;
+}
+
+export async function writeCache(): Promise<void> {
+    for (const modelId of MODEL_IDS) {
+        const url = `${DEVICE_URL}${modelId}/info`;
+        const image = await getJson<ImageJson>(NAME, url);
+
+        if (image?.latest_firmware?.release?.images) {
+            writeCacheJson(getCacheFileName(modelId), image);
+        }
+    }
+}
+
+export async function download(): Promise<void> {
+    for (const modelId of MODEL_IDS) {
+        const url = `${DEVICE_URL}${modelId}/info`;
+        const image = await getJson<ImageJson>(NAME, url);
+
+        // XXX: this is assumed to always be present even for devices that support OTA but without images yet available?
+        if (image?.latest_firmware?.release?.images) {
+            const firmware = image.latest_firmware.release.images['zigbee.ota'];
+
+            if (!firmware) {
+                continue;
+            }
+
+            const firmwareUrl = BASE_URL + firmware.url;
+            const firmwareFileName = firmwareUrl.split('/').pop()!;
+            const cacheFileName = getCacheFileName(modelId);
+
+            if (!isDifferent(image, readCacheJson(cacheFileName))) {
+                console.log(`[${NAME}:${firmwareFileName}] No change from last run.`);
+                continue;
+            }
+
+            writeCacheJson(cacheFileName, image);
+
+            await processFirmwareImage(NAME, firmwareFileName, firmwareUrl, {
+                manufacturerName: [NAME],
+                releaseNotes: BASE_URL + image.latest_firmware.release.changelog,
+            });
+        } else {
+            console.error(`${LOG_PREFIX} No image data for ${modelId}.`);
+            continue;
+        }
+    }
+}

src/autodl/ledvance.ts

@@ -0,0 +1,128 @@
+import {getJson, getLatestImage, readCacheJson, writeCacheJson} from '../common.js';
+import {processFirmwareImage, ProcessFirmwareImageStatus} from '../process_firmware_image.js';
+
+type FirmwareJson = {
+    blob: null;
+    identity: {
+        company: number;
+        product: number;
+        version: {
+            major: number;
+            minor: number;
+            build: number;
+            revision: number;
+        };
+    };
+    releaseNotes: string;
+    /** Ledvance's API docs state the checksum should be `sha_256` but it is actually `shA256` */
+    shA256: string;
+    name: string;
+    productName: string;
+    /**
+     * The fileVersion in hex is included in the fullName between the `/`, e.g.:
+     *   - PLUG COMPACT EU T/032b3674/PLUG_COMPACT_EU_T-0x00D6-0x032B3674-MF_DIS.OTA
+     *   - A19 RGBW/00102428/A19_RGBW_IMG0019_00102428-encrypted.ota
+     */
+    fullName: string;
+    extension: string;
+    released: string;
+    salesRegion: string;
+    length: number;
+};
+type ImagesJson = {
+    firmwares: FirmwareJson[];
+};
+type GroupedImagesJson = Record<string, FirmwareJson[]>;
+
+const NAME = 'Ledvance';
+const LOG_PREFIX = `[${NAME}]`;
+const FIRMWARE_URL = 'https://api.update.ledvance.com/v1/zigbee/firmwares/';
+// const UPDATE_CHECK_URL = 'https://api.update.ledvance.com/v1/zigbee/firmwares/newer';
+// const UPDATE_CHECK_PARAMS = `?company=${manufCode}&product=${imageType}&version=0.0.0`;
+const UPDATE_DOWNLOAD_URL = 'https://api.update.ledvance.com/v1/zigbee/firmwares/download';
+/** XXX: getting 429 after a few downloads, force more throttling. Seems to trigger after around ~20 requests. */
+const FETCH_FAILED_THROTTLE_MS = 60000;
+const FETCH_FAILED_RETRIES = 3;
+
+function groupByProduct(arr: FirmwareJson[]): GroupedImagesJson {
+    return arr.reduce<GroupedImagesJson>((acc, cur) => {
+        acc[cur.identity.product] = [...(acc[cur.identity.product] || []), cur];
+        return acc;
+    }, {});
+}
+
+function sortByReleased(a: FirmwareJson, b: FirmwareJson): number {
+    return a.released < b.released ? -1 : a.released > b.released ? 1 : 0;
+}
+
+function getVersionString(firmware: FirmwareJson): string {
+    const {major, minor, build, revision} = firmware.identity.version;
+
+    return `${major}.${minor}.${build}.${revision}`;
+}
+
+function isDifferent(newData: FirmwareJson, cachedData?: FirmwareJson): boolean {
+    return Boolean(process.env.IGNORE_CACHE) || !cachedData || getVersionString(cachedData) !== getVersionString(newData);
+}
+
+export async function writeCache(): Promise<void> {
+    const images = await getJson<ImagesJson>(NAME, FIRMWARE_URL);
+
+    if (images?.firmwares?.length) {
+        writeCacheJson(NAME, images);
+    }
+}
+
+export async function download(): Promise<void> {
+    const images = await getJson<ImagesJson>(NAME, FIRMWARE_URL);
+
+    if (images?.firmwares?.length) {
+        const cachedData = readCacheJson<ImagesJson | undefined>(NAME);
+        const cachedDataByProduct = cachedData?.firmwares?.length ? groupByProduct(cachedData.firmwares) : undefined;
+        const firmwareByProduct = groupByProduct(images.firmwares);
+
+        for (const product in firmwareByProduct) {
+            const firmware = getLatestImage(firmwareByProduct[product], sortByReleased);
+
+            if (!firmware) {
+                console.error(`${LOG_PREFIX} No image found for ${product}.`);
+                continue;
+            }
+
+            const fileVersionMatch = /\/(\d|\w+)\//.exec(firmware.fullName);
+
+            if (fileVersionMatch == null) {
+                // ignore possible unsupported patterns
+                continue;
+            }
+
+            // const fileVersion = parseInt(fileVersionMatch[1], 16);
+            const firmwareUrl = `${UPDATE_DOWNLOAD_URL}?company=${firmware.identity.company}&product=${firmware.identity.product}&version=${getVersionString(firmware)}`;
+            const firmwareFileName = firmware.fullName.split('/').pop()!;
+
+            if (cachedDataByProduct && !isDifferent(firmware, getLatestImage(cachedDataByProduct[product], sortByReleased))) {
+                console.log(`[${NAME}:${firmwareFileName}] No change from last run.`);
+                continue;
+            }
+
+            for (let i = 0; i < FETCH_FAILED_RETRIES; i++) {
+                const status = await processFirmwareImage(NAME, firmwareFileName, firmwareUrl, {
+                    manufacturerName: [NAME],
+                    // workflow automatically computes sha512
+                    // sha256: firmware.shA256,
+                    releaseNotes: firmware.releaseNotes,
+                });
+
+                if (status === ProcessFirmwareImageStatus.REQUEST_FAILED) {
+                    await new Promise((resolve) => setTimeout(resolve, FETCH_FAILED_THROTTLE_MS));
+                } else {
+                    break;
+                }
+            }
+        }
+
+        writeCacheJson(NAME, images);
+    } else {
+        console.error(`${LOG_PREFIX} No image data.`);
+    }
+}

src/autodl/lixee.ts

@@ -0,0 +1,15 @@
+import * as github from './github.js';
+
+const NAME = 'Lixee';
+const FIRMWARE_URL = 'https://api.github.com/repos/fairecasoimeme/Zlinky_TIC/releases';
+/** @see https://github.com/fairecasoimeme/Zlinky_TIC?tab=readme-ov-file#route-or-limited-route-from-v7 */
+const FIRMWARE_EXT = '.ota';
+const FIRMWARE_LIMITED = `limited${FIRMWARE_EXT}`;
+
+export async function writeCache(): Promise<void> {
+    await github.writeCache(NAME, FIRMWARE_URL);
+}
+
+export async function download(): Promise<void> {
+    await github.download(NAME, FIRMWARE_URL, [(a): boolean => a.name.endsWith(FIRMWARE_EXT), (a): boolean => a.name.endsWith(FIRMWARE_LIMITED)]);
+}

src/autodl/salus.ts

@@ -0,0 +1,55 @@
+import {getJson, readCacheJson, writeCacheJson} from '../common.js';
+import {processFirmwareImage} from '../process_firmware_image.js';
+
+type ImageJson = {
+    model: string;
+    version: string;
+    url: string;
+};
+type ImagesJson = {
+    versions: ImageJson[];
+};
+
+const NAME = 'Salus';
+const LOG_PREFIX = `[${NAME}]`;
+const FIRMWARE_URL = 'https://eu.salusconnect.io/demo/default/status/firmware?timestamp=0';
+
+function findInCache(image: ImageJson, cachedData?: ImagesJson): ImageJson | undefined {
+    return cachedData?.versions?.find((d) => d.model == image.model);
+}
+
+function isDifferent(newData: ImageJson, cachedData?: ImageJson): boolean {
+    return Boolean(process.env.IGNORE_CACHE) || !cachedData || cachedData.version !== newData.version;
+}
+
+export async function writeCache(): Promise<void> {
+    const images = await getJson<ImagesJson>(NAME, FIRMWARE_URL);
+
+    if (images?.versions?.length) {
+        writeCacheJson(NAME, images);
+    }
+}
+
+export async function download(): Promise<void> {
+    const images = await getJson<ImagesJson>(NAME, FIRMWARE_URL);
+
+    if (images?.versions?.length) {
+        const cachedData = readCacheJson<ImagesJson | undefined>(NAME);
+
+        for (const image of images.versions) {
+            const archiveUrl = image.url; //.replace(/^http:\/\//, 'https://');
+            const archiveFileName = archiveUrl.split('/').pop()!;
+
+            if (!isDifferent(image, findInCache(image, cachedData))) {
+                console.log(`[${NAME}:${archiveFileName}] No change from last run.`);
+                continue;
+            }
+
+            await processFirmwareImage(NAME, archiveFileName, archiveUrl, {manufacturerName: [NAME]}, true, (fileName) => fileName.endsWith('.ota'));
+        }
+
+        writeCacheJson(NAME, images);
+    } else {
+        console.error(`${LOG_PREFIX} No image data.`);
+    }
+}

src/autodl/ubisys.ts

@@ -0,0 +1,104 @@
+import url from 'url';
+
+import {getLatestImage, getText, readCacheJson, writeCacheJson} from '../common.js';
+import {processFirmwareImage} from '../process_firmware_image.js';
+
+type Image = {
+    fileName: string;
+    imageType: string;
+    hardwareVersionMin: number;
+    hardwareVersionMax: number;
+    fileVersion: number;
+};
+type GroupedImages = {
+    [k: string]: Image[];
+};
+
+const NAME = 'Ubisys';
+const LOG_PREFIX = `[${NAME}]`;
+const FIRMWARE_HTML_URL = 'http://fwu.ubisys.de/smarthome/OTA/release/index';
+
+function groupByImageType(arr: Image[]): GroupedImages {
+    return arr.reduce<GroupedImages>((acc, cur) => {
+        acc[cur.imageType] = [...(acc[cur.imageType] || []), cur];
+        return acc;
+    }, {});
+}
+
+function sortByFileVersion(a: Image, b: Image): number {
+    return a.fileVersion < b.fileVersion ? -1 : a.fileVersion > b.fileVersion ? 1 : 0;
+}
+
+function isDifferent(newData: Image, cachedData?: Image): boolean {
+    return Boolean(process.env.IGNORE_CACHE) || !cachedData || cachedData.fileVersion !== newData.fileVersion;
+}
+
+function parseText(pageText: string): Image[] {
+    const lines = pageText.split('\n');
+    const images: Image[] = [];
+
+    for (const line of lines) {
+        // XXX: there are other images on the page that do not match this pattern
+        const imageMatch = /10F2-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{8})\S*ota1?\.zigbee/gi.exec(line);
+
+        if (imageMatch != null) {
+            images.push({
+                fileName: imageMatch[0],
+                imageType: imageMatch[1],
+                hardwareVersionMin: parseInt(imageMatch[2], 16),
+                hardwareVersionMax: parseInt(imageMatch[3], 16),
+                fileVersion: parseInt(imageMatch[4], 16),
+            });
+        }
+    }
+
+    return images;
+}
+
+export async function writeCache(): Promise<void> {
+    const pageText = await getText(NAME, FIRMWARE_HTML_URL);
+
+    if (pageText?.length) {
+        const images = parseText(pageText);
+
+        writeCacheJson(NAME, images);
+    }
+}
+
+export async function download(): Promise<void> {
+    const pageText = await getText(NAME, FIRMWARE_HTML_URL);
+
+    if (pageText?.length) {
+        const images = parseText(pageText);
+        const imagesByType = groupByImageType(images);
+        const cachedData = readCacheJson<Image[] | undefined>(NAME);
+        const cachedDataByType = cachedData ? groupByImageType(cachedData) : undefined;
+
+        for (const imageType in imagesByType) {
+            const image = getLatestImage(imagesByType[imageType], sortByFileVersion);
+
+            if (!image) {
+                console.error(`${LOG_PREFIX} No image found for ${imageType}.`);
+                continue;
+            }
+
+            if (cachedDataByType && !isDifferent(image, getLatestImage(cachedDataByType[imageType], sortByFileVersion))) {
+                console.log(`[${NAME}:${image.fileName}] No change from last run.`);
+                continue;
+            }
+
+            // NOTE: removes `index` from url
+            const firmwareUrl = url.resolve(FIRMWARE_HTML_URL, image.fileName);
+
+            await processFirmwareImage(NAME, image.fileName, firmwareUrl, {
+                manufacturerName: [NAME],
+                hardwareVersionMin: image.hardwareVersionMin,
+                hardwareVersionMax: image.hardwareVersionMax,
+            });
+        }
+
+        writeCacheJson(NAME, images);
+    } else {
+        console.error(`${LOG_PREFIX} No image data.`);
+    }
+}

src/autodl/xyzroe.ts

@@ -0,0 +1,13 @@
+import * as github from './github.js';
+
+const NAME = 'Xyzroe';
+const FIRMWARE_URL = 'https://api.github.com/repos/xyzroe/ZigUSB_C6/releases';
+const FIRMWARE_EXT = '.ota';
+
+export async function writeCache(): Promise<void> {
+    await github.writeCache(NAME, FIRMWARE_URL);
+}
+
+export async function download(): Promise<void> {
+    await github.download(NAME, FIRMWARE_URL, [(a): boolean => a.name.endsWith(FIRMWARE_EXT)]);
+}

src/common.ts

@@ -0,0 +1,406 @@
+import type {ExtraMetas, ExtraMetasWithFileName, ImageHeader, RepoImageMeta} from './types';
+
+import assert from 'assert';
+import {exec} from 'child_process';
+import {createHash} from 'crypto';
+import {existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync} from 'fs';
+import path from 'path';
+
+export const UPGRADE_FILE_IDENTIFIER = Buffer.from([0x1e, 0xf1, 0xee, 0x0b]);
+export const BASE_REPO_URL = `https://github.com/Koenkk/zigbee-OTA/raw/`;
+export const REPO_BRANCH = 'master';
+/** Images used by OTA upgrade process */
+export const BASE_IMAGES_DIR = 'images';
+/** Images used by OTA downgrade process */
+export const PREV_IMAGES_DIR = 'images1';
+/** Manifest used by OTA upgrade process */
+export const BASE_INDEX_MANIFEST_FILENAME = 'index.json';
+/** Manifest used by OTA downgrade process */
+export const PREV_INDEX_MANIFEST_FILENAME = 'index1.json';
+export const CACHE_DIR = '.cache';
+export const TMP_DIR = 'tmp';
+/**
+ * 'ikea_new' first, to prioritize downloads from new URL
+ */
+export const ALL_AUTODL_MANUFACTURERS = ['gmmts', 'ikea_new', 'ikea', 'inovelli', 'jethome', 'ledvance', 'lixee', 'salus', 'ubisys', 'xyzroe'];
+
+export async function execute(command: string): Promise<string> {
+    return await new Promise((resolve, reject) => {
+        exec(command, (error, stdout) => {
+            if (error) {
+                reject(error);
+            } else {
+                resolve(stdout);
+            }
+        });
+    });
+}
+
+export function primitivesArrayEquals(a: (string | number | boolean)[], b: (string | number | boolean)[]): boolean {
+    return a.length === b.length && a.every((val, index) => val === b[index]);
+}
+
+export function computeSHA512(buffer: Buffer): string {
+    const hash = createHash('sha512');
+
+    hash.update(buffer);
+
+    return hash.digest('hex');
+}
+
+export function getOutDir(folderName: string, basePath: string = BASE_IMAGES_DIR): string {
+    const outDir = path.join(basePath, folderName);
+
+    if (!existsSync(outDir)) {
+        mkdirSync(outDir, {recursive: true});
+    }
+
+    return outDir;
+}
+
+export function getRepoFirmwareFileUrl(folderName: string, fileName: string, basePath: string = BASE_IMAGES_DIR): string {
+    return BASE_REPO_URL + path.posix.join(REPO_BRANCH, basePath, folderName, fileName);
+}
+
+export function writeManifest(fileName: string, firmwareList: RepoImageMeta[]): void {
+    writeFileSync(fileName, JSON.stringify(firmwareList, undefined, 2), 'utf8');
+}
+
+export function readManifest(fileName: string): RepoImageMeta[] {
+    return JSON.parse(readFileSync(fileName, 'utf8'));
+}
+
+export function writeCacheJson<T>(fileName: string, contents: T, basePath: string = CACHE_DIR): void {
+    writeFileSync(path.join(basePath, `${fileName}.json`), JSON.stringify(contents), 'utf8');
+}
+
+export function readCacheJson<T>(fileName: string, basePath: string = CACHE_DIR): T {
+    const filePath = path.join(basePath, `${fileName}.json`);
+
+    return existsSync(filePath) ? JSON.parse(readFileSync(filePath, 'utf8')) : undefined;
+}
+
+export function parseImageHeader(buffer: Buffer): ImageHeader {
+    try {
+        const header: ImageHeader = {
+            otaUpgradeFileIdentifier: buffer.subarray(0, 4),
+            otaHeaderVersion: buffer.readUInt16LE(4),
+            otaHeaderLength: buffer.readUInt16LE(6),
+            otaHeaderFieldControl: buffer.readUInt16LE(8),
+            manufacturerCode: buffer.readUInt16LE(10),
+            imageType: buffer.readUInt16LE(12),
+            fileVersion: buffer.readUInt32LE(14),
+            zigbeeStackVersion: buffer.readUInt16LE(18),
+            otaHeaderString: buffer.toString('utf8', 20, 52),
+            totalImageSize: buffer.readUInt32LE(52),
+        };
+        let headerPos = 56;
+
+        if (header.otaHeaderFieldControl & 1) {
+            header.securityCredentialVersion = buffer.readUInt8(headerPos);
+            headerPos += 1;
+        }
+
+        if (header.otaHeaderFieldControl & 2) {
+            header.upgradeFileDestination = buffer.subarray(headerPos, headerPos + 8);
+            headerPos += 8;
+        }
+
+        if (header.otaHeaderFieldControl & 4) {
+            header.minimumHardwareVersion = buffer.readUInt16LE(headerPos);
+            headerPos += 2;
+            header.maximumHardwareVersion = buffer.readUInt16LE(headerPos);
+            headerPos += 2;
+        }
+
+        assert(UPGRADE_FILE_IDENTIFIER.equals(header.otaUpgradeFileIdentifier), `Invalid upgrade file identifier`);
+
+        return header;
+    } catch (error) {
+        throw new Error(`Not a valid OTA file (${(error as Error).message}).`);
+    }
+}
+
+/**
+ * Adapted from zigbee-herdsman-converters logic
+ */
+export function findMatchImage(
+    image: ImageHeader,
+    imageList: RepoImageMeta[],
+    extraMetas: ExtraMetas,
+): [index: number, image: RepoImageMeta | undefined] {
+    const imageIndex = imageList.findIndex(
+        (i) =>
+            i.imageType === image.imageType &&
+            i.manufacturerCode === image.manufacturerCode &&
+            (!i.minFileVersion || image.fileVersion >= i.minFileVersion) &&
+            (!i.maxFileVersion || image.fileVersion <= i.maxFileVersion) &&
+            i.modelId === extraMetas.modelId &&
+            (!(i.manufacturerName && extraMetas.manufacturerName) || primitivesArrayEquals(i.manufacturerName, extraMetas.manufacturerName)),
+    );
+
+    return [imageIndex, imageIndex === -1 ? undefined : imageList[imageIndex]];
+}
+
+export function changeRepoUrl(repoUrl: string, fromDir: string, toDir: string): string {
+    return repoUrl.replace(path.posix.join(REPO_BRANCH, fromDir), path.posix.join(REPO_BRANCH, toDir));
+}
+
+export async function getJson<T>(manufacturer: string, pageUrl: string): Promise<T | undefined> {
+    const response = await fetch(pageUrl);
+
+    if (!response.ok || !response.body) {
+        console.error(`[${manufacturer}] Invalid response from ${pageUrl} status=${response.status}.`);
+        return;
+    }
+
+    return (await response.json()) as T;
+}
+
+export async function getText(manufacturer: string, pageUrl: string): Promise<string | undefined> {
+    const response = await fetch(pageUrl);
+
+    if (!response.ok || !response.body) {
+        console.error(`[${manufacturer}] Invalid response from ${pageUrl} status=${response.status}.`);
+        return;
+    }
+
+    return await response.text();
+}
+
+export function getLatestImage<T>(list: T[], compareFn: (a: T, b: T) => number): T | undefined {
+    const sortedList = list.sort(compareFn);
+
+    return sortedList.slice(0, sortedList.length > 1 && process.env.PREV ? -1 : undefined).pop();
+}
+
+export const enum ParsedImageStatus {
+    NEW = 0,
+    NEWER = 1,
+    OLDER = 2,
+    IDENTICAL = 3,
+}
+
+export function getParsedImageStatus(parsedImage: ImageHeader, match?: RepoImageMeta): ParsedImageStatus {
+    if (match) {
+        if (match.fileVersion > parsedImage.fileVersion) {
+            return ParsedImageStatus.OLDER;
+        } else if (match.fileVersion < parsedImage.fileVersion) {
+            return ParsedImageStatus.NEWER;
+        } else {
+            return ParsedImageStatus.IDENTICAL;
+        }
+    } else {
+        return ParsedImageStatus.NEW;
+    }
+}
+
+/**
+ * Prevent irrelevant metas from being added to manifest.
+ *
+ * NOTE: fileName should be deleted before adding to manifest for consistency (always use original file name).
+ * @param metas
+ * @returns
+ */
+export function getValidMetas(metas: Partial<ExtraMetas & ExtraMetasWithFileName & RepoImageMeta>, ignoreFileName: boolean): ExtraMetasWithFileName {
+    const validMetas: ExtraMetasWithFileName = {};
+
+    if (!ignoreFileName) {
+        if (metas.fileName != undefined) {
+            if (typeof metas.fileName != 'string') {
+                throw new Error(`Invalid format for 'fileName', expected 'string' type.`);
+            }
+
+            validMetas.fileName = metas.fileName;
+        }
+    }
+
+    if (metas.originalUrl != undefined) {
+        if (typeof metas.originalUrl != 'string') {
+            throw new Error(`Invalid format for 'originalUrl', expected 'string' type.`);
+        }
+
+        validMetas.originalUrl = metas.originalUrl;
+    }
+
+    if (metas.force != undefined) {
+        if (typeof metas.force != 'boolean') {
+            throw new Error(`Invalid format for 'force', expected 'boolean' type.`);
+        }
+
+        validMetas.force = metas.force;
+    }
+
+    if (metas.hardwareVersionMax != undefined) {
+        if (typeof metas.hardwareVersionMax != 'number') {
+            throw new Error(`Invalid format for 'hardwareVersionMax', expected 'number' type.`);
+        }
+
+        validMetas.hardwareVersionMax = metas.hardwareVersionMax;
+    }
+
+    if (metas.hardwareVersionMin != undefined) {
+        if (typeof metas.hardwareVersionMin != 'number') {
+            throw new Error(`Invalid format for 'hardwareVersionMin', expected 'number' type.`);
+        }
+
+        validMetas.hardwareVersionMin = metas.hardwareVersionMin;
+    }
+
+    if (metas.manufacturerName != undefined) {
+        if (!Array.isArray(metas.manufacturerName) || metas.manufacturerName.length < 1 || metas.manufacturerName.some((m) => typeof m != 'string')) {
+            throw new Error(`Invalid format for 'manufacturerName', expected 'array of string' type.`);
+        }
+
+        validMetas.manufacturerName = metas.manufacturerName;
+    }
+
+    if (metas.maxFileVersion != undefined) {
+        if (typeof metas.maxFileVersion != 'number') {
+            throw new Error(`Invalid format for 'maxFileVersion', expected 'number' type.`);
+        }
+
+        validMetas.maxFileVersion = metas.maxFileVersion;
+    }
+
+    if (metas.minFileVersion != undefined) {
+        if (typeof metas.minFileVersion != 'number') {
+            throw new Error(`Invalid format for 'minFileVersion', expected 'number' type.`);
+        }
+
+        validMetas.minFileVersion = metas.minFileVersion;
+    }
+
+    if (metas.modelId != undefined) {
+        if (typeof metas.modelId != 'string') {
+            throw new Error(`Invalid format for 'modelId', expected 'string' type.`);
+        }
+
+        validMetas.modelId = metas.modelId;
+    }
+
+    if (metas.releaseNotes != undefined) {
+        if (typeof metas.releaseNotes != 'string') {
+            throw new Error(`Invalid format for 'releaseNotes', expected 'string' type.`);
+        }
+
+        validMetas.releaseNotes = metas.releaseNotes;
+    }
+
+    return validMetas;
+}
+
+export function addImageToPrev(
+    logPrefix: string,
+    isNewer: boolean,
+    prevManifest: RepoImageMeta[],
+    prevMatchIndex: number,
+    prevMatch: RepoImageMeta,
+    prevOutDir: string,
+    firmwareFileName: string,
+    manufacturer: string,
+    parsedImage: ImageHeader,
+    firmwareBuffer: Buffer,
+    originalUrl: string | undefined,
+    extraMetas: ExtraMetas,
+    onBeforeManifestPush: () => void,
+): void {
+    console.log(`${logPrefix} Base manifest has higher version. Adding to prev instead.`);
+
+    if (isNewer) {
+        console.log(`${logPrefix} Removing prev image.`);
+        prevManifest.splice(prevMatchIndex, 1);
+
+        // make sure fileName exists for migration from old system
+        const prevFileName = prevMatch.fileName ? prevMatch.fileName : prevMatch.url.split('/').pop()!;
+
+        rmSync(path.join(prevOutDir, prevFileName), {force: true});
+    }
+
+    onBeforeManifestPush();
+    prevManifest.push({
+        fileName: firmwareFileName,
+        fileVersion: parsedImage.fileVersion,
+        fileSize: parsedImage.totalImageSize,
+        originalUrl,
+        url: getRepoFirmwareFileUrl(manufacturer, firmwareFileName, PREV_IMAGES_DIR),
+        imageType: parsedImage.imageType,
+        manufacturerCode: parsedImage.manufacturerCode,
+        sha512: computeSHA512(firmwareBuffer),
+        otaHeaderString: parsedImage.otaHeaderString,
+        ...extraMetas,
+    });
+}
+
+export function addImageToBase(
+    logPrefix: string,
+    isNewer: boolean,
+    prevManifest: RepoImageMeta[],
+    prevOutDir: string,
+    baseManifest: RepoImageMeta[],
+    baseMatchIndex: number,
+    baseMatch: RepoImageMeta,
+    baseOutDir: string,
+    firmwareFileName: string,
+    manufacturer: string,
+    parsedImage: ImageHeader,
+    firmwareBuffer: Buffer,
+    originalUrl: string | undefined,
+    extraMetas: ExtraMetas,
+    onBeforeManifestPush: () => void,
+): void {
+    if (isNewer) {
+        console.log(`${logPrefix} Base manifest has older version ${baseMatch.fileVersion}. Replacing with ${parsedImage.fileVersion}.`);
+
+        const [prevMatchIndex, prevMatch] = findMatchImage(parsedImage, prevManifest, extraMetas);
+        const prevStatus = getParsedImageStatus(parsedImage, prevMatch);
+
+        if (prevStatus !== ParsedImageStatus.OLDER && prevStatus !== ParsedImageStatus.NEW) {
+            console.warn(`${logPrefix} Base image is new/newer but prev image is not older/non-existing.`);
+        }
+
+        if (prevStatus !== ParsedImageStatus.NEW) {
+            console.log(`${logPrefix} Removing prev image.`);
+            prevManifest.splice(prevMatchIndex, 1);
+
+            // make sure fileName exists for migration from old system
+            const prevFileName = prevMatch!.fileName ? prevMatch!.fileName : prevMatch!.url.split('/').pop()!;
+
+            rmSync(path.join(prevOutDir, prevFileName), {force: true});
+        }
+
+        // relocate base to prev
+        // make sure fileName exists for migration from old system
+        const baseFileName = baseMatch.fileName ? baseMatch.fileName : baseMatch.url.split('/').pop()!;
+        const baseFilePath = path.join(baseOutDir, baseFileName);
+
+        // if for some reason the file is no longer present (should not happen), don't add it to prev since link is broken
+        if (existsSync(baseFilePath)) {
+            renameSync(baseFilePath, path.join(prevOutDir, baseFileName));
+
+            baseMatch!.url = changeRepoUrl(baseMatch.url, BASE_IMAGES_DIR, PREV_IMAGES_DIR);
+
+            prevManifest.push(baseMatch);
+        } else {
+            console.error(`${logPrefix} Image file '${baseFilePath}' does not exist. Not moving to prev.`);
+        }
+
+        baseManifest.splice(baseMatchIndex, 1);
+    } else {
+        console.log(`${logPrefix} Base manifest does not have version ${parsedImage.fileVersion}. Adding.`);
+    }
+
+    onBeforeManifestPush();
+    baseManifest.push({
+        fileName: firmwareFileName,
+        fileVersion: parsedImage.fileVersion,
+        fileSize: parsedImage.totalImageSize,
+        originalUrl,
+        url: getRepoFirmwareFileUrl(manufacturer, firmwareFileName, BASE_IMAGES_DIR),
+        imageType: parsedImage.imageType,
+        manufacturerCode: parsedImage.manufacturerCode,
+        sha512: computeSHA512(firmwareBuffer),
+        otaHeaderString: parsedImage.otaHeaderString,
+        ...extraMetas,
+    });
+}

src/ghw_concat_cacerts.ts

@@ -0,0 +1,29 @@
+import type CoreApi from '@actions/core';
+import type {Context} from '@actions/github/lib/context';
+import type {Octokit} from '@octokit/rest';
+
+import {readdirSync, readFileSync, writeFileSync} from 'fs';
+import path from 'path';
+
+export const CACERTS_DIR = 'cacerts';
+export const CACERTS_CONCAT_FILEPATH = 'cacerts.pem';
+
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+export async function concatCaCerts(github: Octokit, core: typeof CoreApi, context: Context): Promise<void> {
+    let pemContents: string = '';
+
+    for (const pem of readdirSync(CACERTS_DIR)) {
+        if (!pem.endsWith('.pem')) {
+            continue;
+        }
+
+        core.startGroup(pem);
+
+        pemContents += readFileSync(path.join(CACERTS_DIR, pem), 'utf8');
+        pemContents += '\n';
+
+        core.endGroup();
+    }
+
+    writeFileSync(CACERTS_CONCAT_FILEPATH, pemContents, 'utf8');
+}

src/ghw_create_autodl_release.ts

@@ -0,0 +1,89 @@
+import type CoreApi from '@actions/core';
+import type {Context} from '@actions/github/lib/context';
+import type {Octokit} from '@octokit/rest';
+
+import {BASE_IMAGES_DIR, BASE_INDEX_MANIFEST_FILENAME, execute, PREV_IMAGES_DIR, PREV_INDEX_MANIFEST_FILENAME, readManifest} from './common.js';
+import {RepoImageMeta} from './types.js';
+
+// about 3 lines
+const MAX_RELEASE_NOTES_LENGTH = 380;
+
+function findReleaseNotes(imagePath: string, manifest: RepoImageMeta[]): string | undefined {
+    const metas = manifest.find((m) => m.url.endsWith(imagePath));
+
+    return metas?.releaseNotes;
+}
+
+function listItemWithReleaseNotes(imagePath: string, releaseNotes?: string): string {
+    let listItem = `* ${imagePath}`;
+
+    if (releaseNotes) {
+        let notes = releaseNotes.replace(/[#*\r\n]+/g, '').replaceAll('-', '|');
+
+        if (notes.length > MAX_RELEASE_NOTES_LENGTH) {
+            notes = `${notes.slice(0, MAX_RELEASE_NOTES_LENGTH)}...`;
+        }
+
+        listItem += `
+  - ${notes}`;
+    }
+
+    return listItem;
+}
+
+export async function createAutodlRelease(github: Octokit, core: typeof CoreApi, context: Context): Promise<void> {
+    const tagName = new Date().toISOString().replace(/[:.]/g, '');
+    // --exclude-standard => Add the standard Git exclusions: .git/info/exclude, .gitignore in each directory, and the user’s global exclusion file.
+    // --others => Show other (i.e. untracked) files in the output.
+    // -z => \0 line termination on output and do not quote filenames.
+    const upgradeImagesStr = await execute(`git ls-files --others --exclude-standard --modified -z ${BASE_IMAGES_DIR}`);
+    const downgradeImagesStr = await execute(`git ls-files --others --exclude-standard --modified -z ${PREV_IMAGES_DIR}`);
+
+    core.debug(`git ls-files for ${BASE_IMAGES_DIR}: ${upgradeImagesStr}`);
+    core.debug(`git ls-files for ${PREV_IMAGES_DIR}: ${downgradeImagesStr}`);
+
+    // -1 to remove empty string at end due to \0 termination
+    const upgradeImages = upgradeImagesStr.split('\0').slice(0, -1);
+    const downgradeImages = downgradeImagesStr.split('\0').slice(0, -1);
+
+    core.info(`Upgrade Images List: ${upgradeImages}`);
+    core.info(`Downgrade Images List: ${downgradeImages}`);
+
+    const baseManifest = readManifest(BASE_INDEX_MANIFEST_FILENAME);
+    const prevManifest = readManifest(PREV_INDEX_MANIFEST_FILENAME);
+
+    let body: string | undefined;
+
+    if (upgradeImages.length > 0 || downgradeImages.length > 0) {
+        body = '';
+
+        if (upgradeImages.length > 0) {
+            const listWithReleaseNotes = upgradeImages.map((v) => listItemWithReleaseNotes(v, findReleaseNotes(v, baseManifest)));
+            body += `## New upgrade images from automatic download:
+${listWithReleaseNotes.join('\n')}
+
+`;
+        }
+
+        if (downgradeImages.length > 0) {
+            const listWithReleaseNotes = downgradeImages.map((v) => listItemWithReleaseNotes(v, findReleaseNotes(v, prevManifest)));
+            body += `## New downgrade images from automatic download:
+${listWithReleaseNotes.join('\n')}
+
+`;
+        }
+    }
+
+    await github.rest.repos.createRelease({
+        owner: context.repo.owner,
+        repo: context.repo.repo,
+        tag_name: tagName,
+        name: tagName,
+        body,
+        draft: false,
+        prerelease: false,
+        // get changes from PRs
+        generate_release_notes: true,
+        make_latest: 'true',
+    });
+}

src/ghw_create_pr_to_default.ts

@@ -0,0 +1,57 @@
+import type CoreApi from '@actions/core';
+import type {Context} from '@actions/github/lib/context';
+import type {Octokit} from '@octokit/rest';
+
+import assert from 'assert';
+
+const IGNORE_OTA_WORKFLOW_LABEL = 'ignore-ota-workflow';
+
+export async function createPRToDefault(
+    github: Octokit,
+    core: typeof CoreApi,
+    context: Context,
+    fromBranchName: string,
+    title: string,
+): Promise<void> {
+    assert(context.payload.repository);
+    assert(fromBranchName);
+    assert(title);
+
+    const base = context.payload.repository.default_branch;
+
+    try {
+        const createdPRResult = await github.rest.pulls.create({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            head: fromBranchName,
+            base,
+            title,
+        });
+
+        await github.rest.issues.addLabels({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: createdPRResult.data.number,
+            labels: [IGNORE_OTA_WORKFLOW_LABEL],
+        });
+
+        core.notice(`Created pull request #${createdPRResult.data.number} from branch ${fromBranchName}.`);
+    } catch (error) {
+        if (error instanceof Error) {
+            if (error.message.includes(`No commits between ${base} and ${fromBranchName}`)) {
+                await github.rest.git.deleteRef({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    ref: `heads/${fromBranchName}`,
+                });
+
+                core.notice(`Nothing needed re-processing.`);
+
+                // don't fail if no commits
+                return;
+            }
+        }
+
+        throw error;
+    }
+}

src/ghw_overwrite_cache.ts

@@ -0,0 +1,41 @@
+import type CoreApi from '@actions/core';
+import type {Context} from '@actions/github/lib/context';
+import type {Octokit} from '@octokit/rest';
+
+import {existsSync, mkdirSync} from 'fs';
+
+import {ALL_AUTODL_MANUFACTURERS, CACHE_DIR} from './common.js';
+
+export async function overwriteCache(github: Octokit, core: typeof CoreApi, context: Context, manufacturersCSV?: string): Promise<void> {
+    if (!existsSync(CACHE_DIR)) {
+        mkdirSync(CACHE_DIR, {recursive: true});
+    }
+
+    const manufacturers = manufacturersCSV ? manufacturersCSV.trim().split(',') : ALL_AUTODL_MANUFACTURERS;
+
+    for (const manufacturer of manufacturers) {
+        // ignore empty strings
+        if (!manufacturer) {
+            continue;
+        }
+
+        if (!ALL_AUTODL_MANUFACTURERS.includes(manufacturer)) {
+            core.error(`Ignoring invalid manufacturer '${manufacturer}'. Expected any of: ${ALL_AUTODL_MANUFACTURERS}.`);
+            continue;
+        }
+
+        const {writeCache} = await import(`./${manufacturer}.js`);
+
+        core.startGroup(manufacturer);
+        core.info(`[${manufacturer}] Writing cache...`);
+
+        try {
+            await writeCache();
+        } catch (error) {
+            core.error((error as Error).message);
+            core.debug((error as Error).stack!);
+        }
+
+        core.endGroup();
+    }
+}

src/ghw_reprocess_all_images.ts

@@ -0,0 +1,398 @@
+import type CoreApi from '@actions/core';
+import type {Context} from '@actions/github/lib/context';
+import type {Octokit} from '@octokit/rest';
+
+import type {RepoImageMeta} from './types';
+
+import {existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync} from 'fs';
+import path from 'path';
+
+import {
+    addImageToBase,
+    addImageToPrev,
+    BASE_IMAGES_DIR,
+    BASE_INDEX_MANIFEST_FILENAME,
+    BASE_REPO_URL,
+    computeSHA512,
+    findMatchImage,
+    getOutDir,
+    getParsedImageStatus,
+    getRepoFirmwareFileUrl,
+    getValidMetas,
+    ParsedImageStatus,
+    parseImageHeader,
+    PREV_IMAGES_DIR,
+    PREV_INDEX_MANIFEST_FILENAME,
+    readManifest,
+    REPO_BRANCH,
+    UPGRADE_FILE_IDENTIFIER,
+    writeManifest,
+} from './common.js';
+
+/** These are now handled by autodl */
+const IGNORE_3RD_PARTIES = ['https://github.com/fairecasoimeme/', 'https://github.com/xyzroe/'];
+
+const DIR_3RD_PARTIES = {
+    'https://otau.meethue.com/': 'Hue',
+    'https://images.tuyaeu.com/': 'Tuya',
+    'https://tr-zha.s3.amazonaws.com/': 'ThirdReality',
+    // NOTE: no longer valid / unable to access via script
+    // 'https://www.elektroimportoren.no/docs/lib/4512772-Firmware-35.ota': 'Namron',
+    // 'https://deconz.dresden-elektronik.de/': 'DresdenElektronik',
+};
+
+export const NOT_IN_BASE_MANIFEST_IMAGES_DIR = 'not-in-manifest-images';
+export const NOT_IN_PREV_MANIFEST_IMAGES_DIR = 'not-in-manifest-images1';
+export const NOT_IN_MANIFEST_FILENAME = 'not-in-manifest.json';
+
+function ignore3rdParty(meta: RepoImageMeta): boolean {
+    for (const ignore of IGNORE_3RD_PARTIES) {
+        if (meta.url.startsWith(ignore)) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+function get3rdPartyDir(meta: RepoImageMeta): string | undefined {
+    for (const key in DIR_3RD_PARTIES) {
+        if (meta.url.startsWith(key)) {
+            return DIR_3RD_PARTIES[key as keyof typeof DIR_3RD_PARTIES];
+        }
+    }
+}
+
+async function download3rdParties(
+    github: Octokit,
+    core: typeof CoreApi,
+    context: Context,
+    /* istanbul ignore next */
+    outDirFinder = get3rdPartyDir,
+): Promise<void> {
+    if (!process.env.NODE_EXTRA_CA_CERTS) {
+        throw new Error(`Download 3rd Parties requires \`NODE_EXTRA_CA_CERTS=cacerts.pem\`.`);
+    }
+
+    const baseManifest = readManifest(BASE_INDEX_MANIFEST_FILENAME);
+    const baseManifestCopy = baseManifest.slice();
+    const prevManifest = readManifest(PREV_INDEX_MANIFEST_FILENAME);
+    let baseImagesAddCount = 0;
+    let prevImagesAddCount = 0;
+
+    for (const meta of baseManifestCopy) {
+        // just in case
+        if (!meta.url) {
+            core.error(`Ignoring malformed ${JSON.stringify(meta)}.`);
+            baseManifest.splice(baseManifest.indexOf(meta), 1);
+            continue;
+        }
+
+        if (meta.url.startsWith(BASE_REPO_URL + REPO_BRANCH)) {
+            core.debug(`Ignoring local URL: ${meta.url}`);
+            continue;
+        }
+
+        // remove itself from base manifest
+        baseManifest.splice(baseManifest.indexOf(meta), 1);
+
+        if (ignore3rdParty(meta)) {
+            core.warning(`Removing ignored '${meta.url}'.`);
+            continue;
+        }
+
+        // reverse add.js logic
+        const fileName = unescape(meta.url.split('/').pop()!);
+        const outDirName = outDirFinder(meta);
+
+        if (outDirName) {
+            core.info(`Downloading 3rd party '${fileName}' into '${outDirName}'`);
+
+            let firmwareFilePath: string | undefined;
+
+            try {
+                const baseOutDir = getOutDir(outDirName, BASE_IMAGES_DIR);
+                const prevOutDir = getOutDir(outDirName, PREV_IMAGES_DIR);
+                const extraMetas = getValidMetas(meta, true);
+
+                core.info(`Extra metas for ${fileName}: ${JSON.stringify(extraMetas)}.`);
+
+                const firmwareFile = await fetch(meta.url);
+
+                if (!firmwareFile.ok || !firmwareFile.body) {
+                    core.error(`Invalid response from ${meta.url} status=${firmwareFile.status}.`);
+                    continue;
+                }
+
+                const firmwareBuffer = Buffer.from(await firmwareFile.arrayBuffer());
+                // make sure to parse from the actual start of the "spec OTA" portion of the file (e.g. Ikea has non-spec meta before)
+                const parsedImage = parseImageHeader(firmwareBuffer.subarray(firmwareBuffer.indexOf(UPGRADE_FILE_IDENTIFIER)));
+                const [baseMatchIndex, baseMatch] = findMatchImage(parsedImage, baseManifest, extraMetas);
+                const statusToBase = getParsedImageStatus(parsedImage, baseMatch);
+
+                switch (statusToBase) {
+                    case ParsedImageStatus.OLDER: {
+                        addImageToPrev(
+                            `[${fileName}]`,
+                            false, // no prev existed before
+                            prevManifest,
+                            -1,
+                            // @ts-expect-error false above prevents issue
+                            undefined,
+                            prevOutDir,
+                            fileName,
+                            outDirName,
+                            parsedImage,
+                            firmwareBuffer,
+                            meta.url,
+                            extraMetas,
+                            () => {
+                                firmwareFilePath = path.join(prevOutDir, fileName);
+
+                                // write before adding to manifest, in case of failure (throw), manifest won't have a broken link
+                                writeFileSync(firmwareFilePath, firmwareBuffer);
+                            },
+                        );
+
+                        prevImagesAddCount++;
+
+                        break;
+                    }
+
+                    case ParsedImageStatus.IDENTICAL: {
+                        core.warning(`Conflict with image at index \`${baseMatchIndex}\`: ${JSON.stringify(baseMatch)}`);
+                        continue;
+                    }
+
+                    case ParsedImageStatus.NEWER:
+                    case ParsedImageStatus.NEW: {
+                        addImageToBase(
+                            `[${fileName}]`,
+                            statusToBase === ParsedImageStatus.NEWER,
+                            prevManifest,
+                            prevOutDir,
+                            baseManifest,
+                            baseMatchIndex,
+                            baseMatch!,
+                            baseOutDir,
+                            fileName,
+                            outDirName,
+                            parsedImage,
+                            firmwareBuffer,
+                            meta.url,
+                            extraMetas,
+                            () => {
+                                firmwareFilePath = path.join(baseOutDir, fileName);
+
+                                // write before adding to manifest, in case of failure (throw), manifest won't have a broken link
+                                writeFileSync(firmwareFilePath, firmwareBuffer);
+                            },
+                        );
+
+                        baseImagesAddCount++;
+
+                        break;
+                    }
+                }
+            } catch (error) {
+                core.error(`Ignoring ${fileName}: ${error}`);
+
+                /* istanbul ignore next */
+                if (firmwareFilePath) {
+                    rmSync(firmwareFilePath, {force: true});
+                }
+
+                continue;
+            }
+        } else {
+            core.warning(`Ignoring '${fileName}' with no out dir specified.`);
+        }
+    }
+
+    writeManifest(PREV_INDEX_MANIFEST_FILENAME, prevManifest);
+    writeManifest(BASE_INDEX_MANIFEST_FILENAME, baseManifest);
+
+    core.info(`Downloaded ${prevImagesAddCount} prev images.`);
+    core.info(`Downloaded ${baseImagesAddCount} base images.`);
+
+    core.info(`Base manifest now contains ${baseManifest.length} images.`);
+    core.info(`Prev manifest now contains ${prevManifest.length} images.`);
+}
+
+function checkImagesAgainstManifests(github: Octokit, core: typeof CoreApi, context: Context, removeNotInManifest: boolean): void {
+    for (const [manifestName, imagesDir, moveDir] of [
+        [PREV_INDEX_MANIFEST_FILENAME, PREV_IMAGES_DIR, NOT_IN_PREV_MANIFEST_IMAGES_DIR],
+        [BASE_INDEX_MANIFEST_FILENAME, BASE_IMAGES_DIR, NOT_IN_BASE_MANIFEST_IMAGES_DIR],
+    ]) {
+        const manifest = readManifest(manifestName);
+        const rewriteManifest: RepoImageMeta[] = [];
+        const missingManifest: RepoImageMeta[] = [];
+
+        core.info(`Checking ${manifestName} (currently ${manifest.length} images)...`);
+
+        for (const subfolderName of readdirSync(imagesDir)) {
+            // skip removal of anything not desired while running jest tests
+            // compare should match data.test.ts > IMAGES_TEST_DIR
+            /* istanbul ignore if */
+            if (process.env.JEST_WORKER_ID && subfolderName !== 'jest-tmp') {
+                continue;
+            }
+
+            const subfolderPath = path.join(imagesDir, subfolderName);
+
+            if (lstatSync(subfolderPath).isDirectory()) {
+                core.startGroup(subfolderPath);
+
+                for (const fileName of readdirSync(subfolderPath)) {
+                    const firmwareFilePath = path.join(subfolderPath, fileName);
+                    const fileRelUrl = path.posix.join(imagesDir, subfolderName, fileName);
+                    // previous add.js used escape() for url property
+                    const escFileRelUrl = escape(fileRelUrl);
+                    // take local images only
+                    const inManifest = manifest.filter(
+                        (m) => m.url.startsWith(BASE_REPO_URL + REPO_BRANCH) && (m.url.endsWith(fileRelUrl) || m.url.endsWith(escFileRelUrl)),
+                    );
+
+                    if (inManifest.length === 0) {
+                        core.warning(`Not found in base manifest: ${firmwareFilePath}.`);
+
+                        if (removeNotInManifest) {
+                            core.error(`Removing ${firmwareFilePath}.`);
+                            rmSync(firmwareFilePath, {force: true});
+                        } else {
+                            const destDirPath = path.join(moveDir, subfolderName);
+
+                            if (!existsSync(destDirPath)) {
+                                mkdirSync(destDirPath, {recursive: true});
+                            }
+
+                            try {
+                                const firmwareBuffer = Buffer.from(readFileSync(firmwareFilePath));
+                                // make sure to parse from the actual start of the "spec OTA" portion of the file (e.g. Ikea has non-spec meta before)
+                                const parsedImage = parseImageHeader(firmwareBuffer.subarray(firmwareBuffer.indexOf(UPGRADE_FILE_IDENTIFIER)));
+
+                                renameSync(firmwareFilePath, path.join(destDirPath, fileName));
+                                missingManifest.push({
+                                    fileName,
+                                    fileVersion: parsedImage.fileVersion,
+                                    fileSize: parsedImage.totalImageSize,
+                                    // originalUrl: meta.url,
+                                    url: getRepoFirmwareFileUrl(subfolderName, fileName, imagesDir),
+                                    imageType: parsedImage.imageType,
+                                    manufacturerCode: parsedImage.manufacturerCode,
+                                    sha512: computeSHA512(firmwareBuffer),
+                                    otaHeaderString: parsedImage.otaHeaderString,
+                                });
+                            } catch (error) {
+                                core.error(`Removing ${firmwareFilePath}: ${error}`);
+                                rmSync(firmwareFilePath, {force: true});
+                            }
+                        }
+                    } else {
+                        if (inManifest.length !== 1) {
+                            core.warning(`[${fileRelUrl}] found multiple times in ${manifestName} manifest:`);
+                            core.warning(JSON.stringify(inManifest, undefined, 2));
+                        }
+
+                        for (const meta of inManifest) {
+                            try {
+                                const firmwareBuffer = Buffer.from(readFileSync(firmwareFilePath));
+                                const extraMetas = getValidMetas(meta, true);
+                                // make sure to parse from the actual start of the "spec OTA" portion of the file (e.g. Ikea has non-spec meta before)
+                                const parsedImage = parseImageHeader(firmwareBuffer.subarray(firmwareBuffer.indexOf(UPGRADE_FILE_IDENTIFIER)));
+                                const [, rewriteMatch] = findMatchImage(parsedImage, rewriteManifest, extraMetas);
+
+                                // only add if not already present
+                                if (!rewriteMatch) {
+                                    rewriteManifest.push({
+                                        fileName,
+                                        fileVersion: parsedImage.fileVersion,
+                                        fileSize: parsedImage.totalImageSize,
+                                        // originalUrl: meta.url,
+                                        url: getRepoFirmwareFileUrl(subfolderName, fileName, imagesDir),
+                                        imageType: parsedImage.imageType,
+                                        manufacturerCode: parsedImage.manufacturerCode,
+                                        sha512: computeSHA512(firmwareBuffer),
+                                        otaHeaderString: parsedImage.otaHeaderString,
+                                        ...extraMetas,
+                                    });
+                                }
+                            } catch (error) {
+                                core.error(`Removing ${firmwareFilePath}: ${error}`);
+                                rmSync(firmwareFilePath, {force: true});
+                            }
+                        }
+                    }
+                }
+
+                core.endGroup();
+            } else {
+                // subfolderName here would actually be the file name
+                throw new Error(`Detected file in ${imagesDir} not in subdirectory: ${subfolderName}.`);
+            }
+        }
+
+        // will not run in case removeNotInManifest is true, since nothing added, `moveDir` will also already have been created
+        if (missingManifest.length > 0) {
+            writeManifest(path.join(moveDir, NOT_IN_MANIFEST_FILENAME), missingManifest);
+
+            core.error(`${missingManifest.length} images not in ${manifestName} manifest.`);
+        }
+
+        writeManifest(manifestName, rewriteManifest);
+
+        core.info(`Rewritten ${manifestName} manifest has ${rewriteManifest.length} images.`);
+    }
+}
+
+/**
+ *
+ * @param github
+ * @param core
+ * @param context
+ * @param removeNotInManifest If false, move images to separate directories
+ * @param skipDownload3rdParties Do not execute the download step
+ * @param downloadOutDirFinder Used mainly for jest tests
+ */
+export async function reProcessAllImages(
+    github: Octokit,
+    core: typeof CoreApi,
+    context: Context,
+    removeNotInManifest: boolean,
+    skipDownload3rdParties: boolean,
+    downloadOutDirFinder = get3rdPartyDir,
+): Promise<void> {
+    if (!removeNotInManifest && existsSync(NOT_IN_BASE_MANIFEST_IMAGES_DIR) && readdirSync(NOT_IN_BASE_MANIFEST_IMAGES_DIR).length > 0) {
+        throw new Error(`${NOT_IN_BASE_MANIFEST_IMAGES_DIR} is not empty. Cannot run.`);
+    }
+
+    if (!removeNotInManifest && existsSync(NOT_IN_PREV_MANIFEST_IMAGES_DIR) && readdirSync(NOT_IN_PREV_MANIFEST_IMAGES_DIR).length > 0) {
+        throw new Error(`${NOT_IN_PREV_MANIFEST_IMAGES_DIR} is not empty. Cannot run.`);
+    }
+
+    /* istanbul ignore if */
+    if (!existsSync(BASE_IMAGES_DIR)) {
+        mkdirSync(BASE_IMAGES_DIR, {recursive: true});
+    }
+
+    /* istanbul ignore if */
+    if (!existsSync(PREV_IMAGES_DIR)) {
+        mkdirSync(PREV_IMAGES_DIR, {recursive: true});
+    }
+
+    /* istanbul ignore if */
+    if (!existsSync(BASE_INDEX_MANIFEST_FILENAME)) {
+        writeManifest(BASE_INDEX_MANIFEST_FILENAME, []);
+    }
+
+    /* istanbul ignore if */
+    if (!existsSync(PREV_INDEX_MANIFEST_FILENAME)) {
+        writeManifest(PREV_INDEX_MANIFEST_FILENAME, []);
+    }
+
+    if (!skipDownload3rdParties) {
+        await download3rdParties(github, core, context, downloadOutDirFinder);
+    }
+
+    checkImagesAgainstManifests(github, core, context, removeNotInManifest);
+}

src/ghw_run_autodl.ts

@@ -0,0 +1,58 @@
+import type CoreApi from '@actions/core';
+import type {Context} from '@actions/github/lib/context';
+import type {Octokit} from '@octokit/rest';
+
+import {existsSync, mkdirSync, rmSync} from 'fs';
+
+import {ALL_AUTODL_MANUFACTURERS, BASE_INDEX_MANIFEST_FILENAME, CACHE_DIR, PREV_INDEX_MANIFEST_FILENAME, TMP_DIR, writeManifest} from './common.js';
+
+export async function runAutodl(github: Octokit, core: typeof CoreApi, context: Context, manufacturersCSV?: string): Promise<void> {
+    const manufacturers = manufacturersCSV ? manufacturersCSV.trim().split(',') : ALL_AUTODL_MANUFACTURERS;
+
+    core.info(`Setup...`);
+
+    if (!existsSync(CACHE_DIR)) {
+        mkdirSync(CACHE_DIR, {recursive: true});
+    }
+
+    if (!existsSync(TMP_DIR)) {
+        mkdirSync(TMP_DIR, {recursive: true});
+    }
+
+    if (!existsSync(BASE_INDEX_MANIFEST_FILENAME)) {
+        writeManifest(BASE_INDEX_MANIFEST_FILENAME, []);
+    }
+
+    if (!existsSync(PREV_INDEX_MANIFEST_FILENAME)) {
+        writeManifest(PREV_INDEX_MANIFEST_FILENAME, []);
+    }
+
+    for (const manufacturer of manufacturers) {
+        // ignore empty strings
+        if (!manufacturer) {
+            continue;
+        }
+
+        if (!ALL_AUTODL_MANUFACTURERS.includes(manufacturer)) {
+            core.error(`Ignoring invalid manufacturer '${manufacturer}'. Expected any of: ${ALL_AUTODL_MANUFACTURERS}.`);
+            continue;
+        }
+
+        const {download} = await import(`./autodl/${manufacturer}.js`);
+
+        core.startGroup(manufacturer);
+
+        try {
+            await download();
+        } catch (error) {
+            core.error((error as Error).message);
+            core.debug((error as Error).stack!);
+        }
+
+        core.endGroup();
+    }
+
+    core.info(`Teardown...`);
+
+    rmSync(TMP_DIR, {recursive: true, force: true});
+}

src/ghw_update_ota_pr.ts

@@ -0,0 +1,307 @@
+import type CoreApi from '@actions/core';
+import type {Context} from '@actions/github/lib/context';
+import type {Octokit} from '@octokit/rest';
+
+import type {ExtraMetas, GHExtraMetas} from './types';
+
+import assert from 'assert';
+import {readFileSync, renameSync} from 'fs';
+import path from 'path';
+
+import {
+    addImageToBase,
+    addImageToPrev,
+    BASE_IMAGES_DIR,
+    BASE_INDEX_MANIFEST_FILENAME,
+    execute,
+    findMatchImage,
+    getOutDir,
+    getParsedImageStatus,
+    getValidMetas,
+    ParsedImageStatus,
+    parseImageHeader,
+    PREV_IMAGES_DIR,
+    PREV_INDEX_MANIFEST_FILENAME,
+    readManifest,
+    UPGRADE_FILE_IDENTIFIER,
+    writeManifest,
+} from './common.js';
+
+const EXTRA_METAS_PR_BODY_START_TAG = '```json';
+const EXTRA_METAS_PR_BODY_END_TAG = '```';
+
+function getFileExtraMetas(extraMetas: GHExtraMetas, fileName: string): ExtraMetas {
+    if (Array.isArray(extraMetas)) {
+        const fileExtraMetas = extraMetas.find((m) => m.fileName === fileName) ?? {};
+        /** @see getValidMetas */
+        delete fileExtraMetas.fileName;
+
+        return fileExtraMetas;
+    }
+
+    // not an array, use same metas for all files
+    return extraMetas;
+}
+
+async function parsePRBodyExtraMetas(github: Octokit, core: typeof CoreApi, context: Context): Promise<GHExtraMetas> {
+    let extraMetas: GHExtraMetas = {};
+
+    if (context.payload.pull_request?.body) {
+        try {
+            const prBody = context.payload.pull_request.body;
+            const metasStart = prBody.indexOf(EXTRA_METAS_PR_BODY_START_TAG);
+            const metasEnd = prBody.lastIndexOf(EXTRA_METAS_PR_BODY_END_TAG);
+
+            if (metasStart !== -1 && metasEnd > metasStart) {
+                const metas = JSON.parse(prBody.slice(metasStart + EXTRA_METAS_PR_BODY_START_TAG.length, metasEnd)) as GHExtraMetas;
+
+                core.info(`Extra metas from PR body:`);
+                core.info(JSON.stringify(metas, undefined, 2));
+
+                if (Array.isArray(metas)) {
+                    extraMetas = [];
+
+                    for (const meta of metas) {
+                        if (!meta.fileName || typeof meta.fileName != 'string') {
+                            core.info(`Ignoring meta in array with missing/invalid fileName:`);
+                            core.info(JSON.stringify(meta, undefined, 2));
+                            continue;
+                        }
+
+                        extraMetas.push(getValidMetas(meta, false));
+                    }
+                } else {
+                    extraMetas = getValidMetas(metas, false);
+                }
+            }
+        } catch (error) {
+            const failureComment = `Invalid extra metas in pull request body: ` + (error as Error).message;
+
+            core.error(failureComment);
+
+            await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: failureComment,
+            });
+
+            throw new Error(failureComment);
+        }
+    }
+
+    return extraMetas;
+}
+
+export async function updateOtaPR(github: Octokit, core: typeof CoreApi, context: Context, fileParam: string): Promise<void> {
+    assert(fileParam, 'No file found in pull request.');
+    assert(context.payload.pull_request, 'Not a pull request');
+
+    const fileParamArr = fileParam.trim().split(',');
+    // take care of empty strings (GH workflow adds a comma at end), ignore files not stored in images dir
+    const fileList = fileParamArr.filter((f) => f.startsWith(`${BASE_IMAGES_DIR}/`));
+
+    assert(fileList.length > 0, 'No image found in pull request.');
+    core.info(`Images in pull request: ${fileList}.`);
+
+    const fileListWrongDir = fileParamArr.filter((f) => f.startsWith(`${PREV_IMAGES_DIR}/`));
+
+    if (fileListWrongDir.length > 0) {
+        const failureComment = `Detected files in 'images1':
+\`\`\`
+${fileListWrongDir.join('\n')}
+\`\`\`
+Please move all files to 'images' (in appropriate subfolders). The pull request will automatically determine the proper location on merge.`;
+
+        await github.rest.issues.createComment({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: context.issue.number,
+            body: failureComment,
+        });
+
+        throw new Error(failureComment);
+    }
+
+    const fileListNoIndex = fileParamArr.filter((f) => f.startsWith(BASE_INDEX_MANIFEST_FILENAME) || f.startsWith(PREV_INDEX_MANIFEST_FILENAME));
+
+    if (fileListNoIndex.length > 0) {
+        const failureComment = `Detected manual changes in ${fileListNoIndex.join(', ')}. Please remove these changes. The pull request will automatically determine the manifests on merge.`;
+
+        await github.rest.issues.createComment({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: context.issue.number,
+            body: failureComment,
+        });
+
+        throw new Error(failureComment);
+    }
+
+    // called at the top, fail early if invalid PR body metas
+    const extraMetas = await parsePRBodyExtraMetas(github, core, context);
+    const baseManifest = readManifest(BASE_INDEX_MANIFEST_FILENAME);
+    const prevManifest = readManifest(PREV_INDEX_MANIFEST_FILENAME);
+
+    for (const file of fileList) {
+        core.startGroup(file);
+        core.info(`Processing '${file}'...`);
+
+        let failureComment: string = '';
+
+        try {
+            const firmwareFileName = path.basename(file);
+            const manufacturer = file.replace(BASE_IMAGES_DIR, '').replace(firmwareFileName, '').replaceAll('/', '').trim();
+
+            if (!manufacturer) {
+                throw new Error(`\`${file}\` should be in its associated manufacturer subfolder.`);
+            }
+
+            const firmwareBuffer = Buffer.from(readFileSync(file));
+            const parsedImage = parseImageHeader(firmwareBuffer.subarray(firmwareBuffer.indexOf(UPGRADE_FILE_IDENTIFIER)));
+
+            core.info(`[${file}] Parsed image header:`);
+            core.info(JSON.stringify(parsedImage, undefined, 2));
+
+            const fileExtraMetas = getFileExtraMetas(extraMetas, firmwareFileName);
+
+            core.info(`[${file}] Extra metas:`);
+            core.info(JSON.stringify(fileExtraMetas, undefined, 2));
+
+            const baseOutDir = getOutDir(manufacturer, BASE_IMAGES_DIR);
+            const prevOutDir = getOutDir(manufacturer, PREV_IMAGES_DIR);
+            const [baseMatchIndex, baseMatch] = findMatchImage(parsedImage, baseManifest, fileExtraMetas);
+            const statusToBase = getParsedImageStatus(parsedImage, baseMatch);
+
+            switch (statusToBase) {
+                case ParsedImageStatus.OLDER: {
+                    // if prev doesn't have a match, move to prev
+                    const [prevMatchIndex, prevMatch] = findMatchImage(parsedImage, prevManifest, fileExtraMetas);
+                    const statusToPrev = getParsedImageStatus(parsedImage, prevMatch);
+
+                    switch (statusToPrev) {
+                        case ParsedImageStatus.OLDER:
+                        case ParsedImageStatus.IDENTICAL: {
+                            failureComment = `Base manifest has higher version:
+\`\`\`json
+${JSON.stringify(baseMatch, undefined, 2)}
+\`\`\`
+and an equal or better match is already present in prev manifest:
+\`\`\`json
+${JSON.stringify(prevMatch, undefined, 2)}
+\`\`\`
+Parsed image header:
+\`\`\`json
+${JSON.stringify(parsedImage, undefined, 2)}
+\`\`\``;
+                            break;
+                        }
+
+                        case ParsedImageStatus.NEWER:
+                        case ParsedImageStatus.NEW: {
+                            addImageToPrev(
+                                `[${file}]`,
+                                statusToPrev === ParsedImageStatus.NEWER,
+                                prevManifest,
+                                prevMatchIndex,
+                                prevMatch!,
+                                prevOutDir,
+                                firmwareFileName,
+                                manufacturer,
+                                parsedImage,
+                                firmwareBuffer,
+                                undefined,
+                                fileExtraMetas,
+                                () => {
+                                    // relocate file to prev
+                                    renameSync(file, file.replace(`${BASE_IMAGES_DIR}/`, `${PREV_IMAGES_DIR}/`));
+                                },
+                            );
+
+                            break;
+                        }
+                    }
+
+                    break;
+                }
+
+                case ParsedImageStatus.IDENTICAL: {
+                    failureComment = `Conflict with image at index \`${baseMatchIndex}\`:
+\`\`\`json
+${JSON.stringify(baseMatch, undefined, 2)}
+\`\`\`
+Parsed image header:
+\`\`\`json
+${JSON.stringify(parsedImage, undefined, 2)}
+\`\`\``;
+                    break;
+                }
+
+                case ParsedImageStatus.NEWER:
+                case ParsedImageStatus.NEW: {
+                    addImageToBase(
+                        `[${file}]`,
+                        statusToBase === ParsedImageStatus.NEWER,
+                        prevManifest,
+                        prevOutDir,
+                        baseManifest,
+                        baseMatchIndex,
+                        baseMatch!,
+                        baseOutDir,
+                        firmwareFileName,
+                        manufacturer,
+                        parsedImage,
+                        firmwareBuffer,
+                        undefined,
+                        fileExtraMetas,
+                        () => {
+                            /* noop */
+                        },
+                    );
+
+                    break;
+                }
+            }
+        } catch (error) {
+            failureComment = (error as Error).message;
+        }
+
+        if (failureComment) {
+            core.error(`[${file}] ` + failureComment);
+            await github.rest.pulls.createReviewComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                pull_number: context.issue.number,
+                body: failureComment,
+                commit_id: context.payload.pull_request.head.sha,
+                path: file,
+                subject_type: 'file',
+            });
+
+            throw new Error(failureComment);
+        }
+
+        core.endGroup();
+    }
+
+    writeManifest(PREV_INDEX_MANIFEST_FILENAME, prevManifest);
+    writeManifest(BASE_INDEX_MANIFEST_FILENAME, baseManifest);
+
+    core.info(`Prev manifest has ${prevManifest.length} images.`);
+    core.info(`Base manifest has ${baseManifest.length} images.`);
+
+    if (!context.payload.pull_request.merged) {
+        const diff = await execute(`git diff`);
+
+        await github.rest.issues.createComment({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: context.issue.number,
+            body: `Merging this pull request will add these changes in a following commit:
+\`\`\`diff
+${diff}
+\`\`\`
+`,
+        });
+    }
+}

src/index.ts

@@ -0,0 +1,9 @@
+export * as common from './common.js';
+export {concatCaCerts} from './ghw_concat_cacerts.js';
+export {createAutodlRelease} from './ghw_create_autodl_release.js';
+export {createPRToDefault} from './ghw_create_pr_to_default.js';
+export {overwriteCache} from './ghw_overwrite_cache.js';
+export {reProcessAllImages} from './ghw_reprocess_all_images.js';
+export {runAutodl} from './ghw_run_autodl.js';
+export {updateOtaPR} from './ghw_update_ota_pr.js';
+export {processFirmwareImage} from './process_firmware_image.js';

src/print_ota_image_header.ts

@@ -0,0 +1,5 @@
+import {readFileSync} from 'fs';
+
+import {parseImageHeader} from './common.js';
+
+console.log(parseImageHeader(readFileSync(process.argv[2])));

src/process_firmware_image.ts

@@ -0,0 +1,221 @@
+import type {ExtraMetas} from './types';
+
+import assert from 'assert';
+import {readdirSync, readFileSync, renameSync, rmSync, writeFileSync} from 'fs';
+import path from 'path';
+
+import {extract} from 'tar';
+
+import {
+    addImageToBase,
+    addImageToPrev,
+    BASE_IMAGES_DIR,
+    BASE_INDEX_MANIFEST_FILENAME,
+    findMatchImage,
+    getOutDir,
+    getParsedImageStatus,
+    ParsedImageStatus,
+    parseImageHeader,
+    PREV_IMAGES_DIR,
+    PREV_INDEX_MANIFEST_FILENAME,
+    readManifest,
+    TMP_DIR,
+    UPGRADE_FILE_IDENTIFIER,
+    writeManifest,
+} from './common.js';
+
+export const enum ProcessFirmwareImageStatus {
+    ERROR = -1,
+    SUCCESS = 0,
+    REQUEST_FAILED = 1,
+    TAR_NO_IMAGE = 2,
+}
+
+async function tarExtract(filePath: string, outDir: string, tarImageFinder: (fileName: string) => boolean): Promise<string> {
+    let outFileName: string | undefined;
+
+    try {
+        console.log(`[${filePath}] Extracting TAR...`);
+
+        await extract({file: filePath, cwd: TMP_DIR});
+
+        for (const file of readdirSync(TMP_DIR)) {
+            const archiveFilePath = path.join(TMP_DIR, file);
+
+            if (tarImageFinder(file)) {
+                outFileName = file;
+                renameSync(archiveFilePath, path.join(outDir, outFileName));
+            } else {
+                rmSync(archiveFilePath, {force: true});
+            }
+        }
+    } catch (error) {
+        console.error(error);
+
+        // force throw below, just in case something crashed in-between this being assigned and the end of the try block
+        outFileName = undefined;
+    }
+
+    // always remove archive file once done
+    rmSync(filePath, {force: true});
+
+    if (!outFileName) {
+        throw new Error(`No image found in ${filePath}.`);
+    }
+
+    return outFileName;
+}
+
+export async function processFirmwareImage(
+    manufacturer: string,
+    firmwareFileName: string,
+    firmwareFileUrl: string,
+    extraMetas: ExtraMetas = {},
+    tar: boolean = false,
+    tarImageFinder?: (fileName: string) => boolean,
+): Promise<ProcessFirmwareImageStatus> {
+    // throttle requests (this is done at the top to ensure always executed)
+    await new Promise((resolve) => setTimeout(resolve, 300));
+
+    let firmwareFilePath: string | undefined;
+    const logPrefix = `[${manufacturer}:${firmwareFileName}]`;
+
+    if (tar && !firmwareFileName.endsWith('.tar.gz')) {
+        // ignore non-archive
+        return ProcessFirmwareImageStatus.TAR_NO_IMAGE;
+    }
+
+    const prevManifest = readManifest(PREV_INDEX_MANIFEST_FILENAME);
+    const baseManifest = readManifest(BASE_INDEX_MANIFEST_FILENAME);
+    const baseOutDir = getOutDir(manufacturer, BASE_IMAGES_DIR);
+    const prevOutDir = getOutDir(manufacturer, PREV_IMAGES_DIR);
+
+    try {
+        const firmwareFile = await fetch(firmwareFileUrl);
+
+        if (!firmwareFile.ok || !firmwareFile.body) {
+            console.error(`${logPrefix} Invalid response from ${firmwareFileUrl} status=${firmwareFile.status}.`);
+            return ProcessFirmwareImageStatus.REQUEST_FAILED;
+        }
+
+        if (tar) {
+            assert(tarImageFinder, `No image finder function supplied for tar.`);
+
+            const archiveBuffer = Buffer.from(await firmwareFile.arrayBuffer());
+            const archiveFilePath = path.join(baseOutDir, firmwareFileName);
+
+            writeFileSync(archiveFilePath, archiveBuffer);
+
+            try {
+                firmwareFileName = await tarExtract(archiveFilePath, baseOutDir, tarImageFinder);
+            } catch {
+                console.error(`${logPrefix} No image found for ${firmwareFileUrl}.`);
+                return ProcessFirmwareImageStatus.TAR_NO_IMAGE;
+            }
+        }
+
+        const firmwareBuffer = tar ? readFileSync(path.join(baseOutDir, firmwareFileName)) : Buffer.from(await firmwareFile.arrayBuffer());
+        // make sure to parse from the actual start of the "spec OTA" portion of the file (e.g. Ikea has non-spec meta before)
+        const parsedImage = parseImageHeader(firmwareBuffer.subarray(firmwareBuffer.indexOf(UPGRADE_FILE_IDENTIFIER)));
+        const [baseMatchIndex, baseMatch] = findMatchImage(parsedImage, baseManifest, extraMetas);
+        const statusToBase = getParsedImageStatus(parsedImage, baseMatch);
+
+        switch (statusToBase) {
+            case ParsedImageStatus.OLDER: {
+                // if prev doesn't have a match, move to prev
+                const [prevMatchIndex, prevMatch] = findMatchImage(parsedImage, prevManifest, extraMetas);
+                const statusToPrev = getParsedImageStatus(parsedImage, prevMatch);
+
+                switch (statusToPrev) {
+                    case ParsedImageStatus.OLDER:
+                    case ParsedImageStatus.IDENTICAL: {
+                        console.log(
+                            `${logPrefix} Base manifest has higher version and an equal or better match is already present in prev manifest. Ignoring.`,
+                        );
+
+                        break;
+                    }
+
+                    case ParsedImageStatus.NEWER:
+                    case ParsedImageStatus.NEW: {
+                        addImageToPrev(
+                            logPrefix,
+                            statusToPrev === ParsedImageStatus.NEWER,
+                            prevManifest,
+                            prevMatchIndex,
+                            prevMatch!,
+                            prevOutDir,
+                            firmwareFileName,
+                            manufacturer,
+                            parsedImage,
+                            firmwareBuffer,
+                            firmwareFileUrl,
+                            extraMetas,
+                            () => {
+                                firmwareFilePath = path.join(prevOutDir, firmwareFileName);
+
+                                // write before adding to manifest, in case of failure (throw), manifest won't have a broken link
+                                writeFileSync(firmwareFilePath, firmwareBuffer);
+                            },
+                        );
+
+                        break;
+                    }
+                }
+
+                break;
+            }
+
+            case ParsedImageStatus.IDENTICAL: {
+                console.log(`${logPrefix} Base manifest already has version ${parsedImage.fileVersion}. Ignoring.`);
+
+                break;
+            }
+
+            case ParsedImageStatus.NEWER:
+            case ParsedImageStatus.NEW: {
+                addImageToBase(
+                    logPrefix,
+                    statusToBase === ParsedImageStatus.NEWER,
+                    prevManifest,
+                    prevOutDir,
+                    baseManifest,
+                    baseMatchIndex,
+                    baseMatch!,
+                    baseOutDir,
+                    firmwareFileName,
+                    manufacturer,
+                    parsedImage,
+                    firmwareBuffer,
+                    firmwareFileUrl,
+                    extraMetas,
+                    () => {
+                        firmwareFilePath = path.join(baseOutDir, firmwareFileName);
+
+                        // write before adding to manifest, in case of failure (throw), manifest won't have a broken link
+                        writeFileSync(firmwareFilePath, firmwareBuffer);
+                    },
+                );
+
+                break;
+            }
+        }
+    } catch (error) {
+        console.error(`${logPrefix} Failed to save firmware file ${firmwareFileName}: ${(error as Error).stack!}.`);
+
+        /* istanbul ignore if */
+        if (firmwareFilePath) {
+            rmSync(firmwareFilePath, {force: true});
+        }
+
+        return ProcessFirmwareImageStatus.ERROR;
+    }
+
+    writeManifest(PREV_INDEX_MANIFEST_FILENAME, prevManifest);
+    writeManifest(BASE_INDEX_MANIFEST_FILENAME, baseManifest);
+
+    console.log(`Prev manifest has ${prevManifest.length} images.`);
+    console.log(`Base manifest has ${baseManifest.length} images.`);
+
+    return ProcessFirmwareImageStatus.SUCCESS;
+}

src/types.ts

@@ -0,0 +1,74 @@
+//-- Copied from ZHC
+export interface Version {
+    imageType: number;
+    manufacturerCode: number;
+    fileVersion: number;
+}
+
+export interface ImageHeader {
+    otaUpgradeFileIdentifier: Buffer;
+    otaHeaderVersion: number;
+    otaHeaderLength: number;
+    otaHeaderFieldControl: number;
+    manufacturerCode: number;
+    imageType: number;
+    fileVersion: number;
+    zigbeeStackVersion: number;
+    otaHeaderString: string;
+    totalImageSize: number;
+    securityCredentialVersion?: number;
+    upgradeFileDestination?: Buffer;
+    minimumHardwareVersion?: number;
+    maximumHardwareVersion?: number;
+}
+
+export interface ImageElement {
+    tagID: number;
+    length: number;
+    data: Buffer;
+}
+
+export interface Image {
+    header: ImageHeader;
+    elements: ImageElement[];
+    raw: Buffer;
+}
+
+export interface ImageInfo {
+    imageType: number;
+    fileVersion: number;
+    manufacturerCode: number;
+}
+
+// XXX: adjusted from ZHC
+export interface ImageMeta {
+    fileVersion: number;
+    fileSize: number;
+    url: string;
+    force?: boolean;
+    sha512: string;
+    otaHeaderString: string;
+    hardwareVersionMin?: number;
+    hardwareVersionMax?: number;
+}
+//-- Copied from ZHC
+
+export interface RepoImageMeta extends ImageInfo, ImageMeta {
+    fileName: string;
+    modelId?: string;
+    manufacturerName?: string[];
+    minFileVersion?: number;
+    maxFileVersion?: number;
+    originalUrl?: string;
+    releaseNotes?: string;
+}
+
+export type ExtraMetas = Omit<
+    RepoImageMeta,
+    'fileName' | 'fileVersion' | 'fileSize' | 'url' | 'imageType' | 'manufacturerCode' | 'sha512' | 'otaHeaderString'
+>;
+export type ExtraMetasWithFileName = Omit<
+    RepoImageMeta,
+    'fileName' | 'fileVersion' | 'fileSize' | 'url' | 'imageType' | 'manufacturerCode' | 'sha512' | 'otaHeaderString'
+> & {fileName?: string};
+export type GHExtraMetas = ExtraMetas | ExtraMetasWithFileName[];