fix: dedupe by manufacturer (#984)

src/autodl/hue.ts

@@ -39,9 +39,9 @@ const DEVICE_TYPE_IDS: string[] = [
     "100b-125",
     // '100b-126',
     "100b-127",
-    // '100b-128',
+    "100b-128",
     "100b-129",
-    // '100b-12a',
+    "100b-12a",
     // '100b-12b',
     // '100b-12c',
     // '100b-12d',
@@ -92,14 +92,21 @@ export async function download(): Promise<void> {
 
         writeCacheJson(cacheFileName, page);
 
-        const image = getLatestImage(page.updates, sortByVersion);
+        page.updates.sort(sortByVersion);
 
-        if (!image) {
-            continue;
+        let previousImage: (typeof page.updates)[number] | undefined;
+
+        for (const image of page.updates) {
+            const firmwareFileName = image.binaryUrl.split("/").pop()!;
+
+            await processFirmwareImage(NAME, firmwareFileName, image.binaryUrl, {
+                releaseNotes: image.versionName
+                    ? `Version: ${image.versionName}${image.releaseNotes ? ` | ${image.releaseNotes}` : ""}`
+                    : image.releaseNotes || undefined,
+                minFileVersion: previousImage?.version,
+            });
+
+            previousImage = image;
         }
-
-        const firmwareFileName = image.binaryUrl.split("/").pop()!;
-
-        await processFirmwareImage(NAME, firmwareFileName, image.binaryUrl, {releaseNotes: image.releaseNotes || undefined});
     }
 }

src/autodl/ikea.ts

@@ -16,9 +16,9 @@ type GatewayImageJson = {
 };
 type DeviceImageJson = {
     fw_binary_url: string;
-    // biome-ignore lint/style/useNamingConvention: <explanation>
+    // biome-ignore lint/style/useNamingConvention: API
     fw_file_version_LSB: number;
-    // biome-ignore lint/style/useNamingConvention: <explanation>
+    // biome-ignore lint/style/useNamingConvention: API
     fw_file_version_MSB: number;
     fw_filesize: number;
     fw_image_type: number;

src/common.ts

@@ -156,7 +156,7 @@ export function findMatchImage(
             extraMetas.hardwareVersionMin === i.hardwareVersionMin &&
             extraMetas.hardwareVersionMax === i.hardwareVersionMax &&
             i.modelId === extraMetas.modelId &&
-            (!i.manufacturerName || (extraMetas.manufacturerName && primitivesArrayEquals(i.manufacturerName, extraMetas.manufacturerName))),
+            (!i.manufacturerName || extraMetas.manufacturerName?.some((v) => i.manufacturerName!.includes(v))),
     );
 
     return [imageIndex, imageIndex === -1 ? undefined : imageList[imageIndex]];
@@ -198,6 +198,21 @@ export function getLatestImage<T>(list: T[] | undefined, compareFn: (a: T, b: T)
     return sortedList.slice(0, sortedList.length > 1 && process.env.PREV ? -1 : undefined).pop();
 }
 
+/** Check if has image in same base path (manufacturer), either by SHA or by spec matching */
+export function hasManufacturerImage(list: RepoImageMeta[], image: RepoImageMeta, extraManufacturerName: string[] | undefined): boolean {
+    const imageBasePath = path.posix.dirname(image.url);
+
+    return list.some(
+        (i) =>
+            path.posix.dirname(i.url) === imageBasePath &&
+            (i.sha512 === image.sha512 ||
+                (i.fileVersion === image.fileVersion &&
+                    i.imageType === image.imageType &&
+                    i.manufacturerCode === image.manufacturerCode &&
+                    (!i.manufacturerName || extraManufacturerName?.some((v) => i.manufacturerName!.includes(v))))),
+    );
+}
+
 export enum ParsedImageStatus {
     New = 0,
     Newer = 1,
@@ -337,6 +352,23 @@ export function addImageToPrev(
 ): void {
     console.log(`${logPrefix} Base manifest has higher version. Adding to prev instead.`);
 
+    const newMetas: RepoImageMeta = {
+        fileName: firmwareFileName,
+        fileVersion: parsedImage.fileVersion,
+        fileSize: parsedImage.totalImageSize,
+        originalUrl,
+        url: getRepoFirmwareFileUrl(manufacturer, firmwareFileName, PREV_IMAGES_DIR),
+        imageType: parsedImage.imageType,
+        manufacturerCode: parsedImage.manufacturerCode,
+        sha512: computeSHA512(firmwareBuffer),
+        otaHeaderString: parsedImage.otaHeaderString.replaceAll("\u0000", ""),
+        ...extraMetas,
+    };
+
+    if (hasManufacturerImage(prevManifest, newMetas, extraMetas.manufacturerName)) {
+        throw new Error("Image already present for manufacturer");
+    }
+
     if (isNewer) {
         console.log(`${logPrefix} Removing prev image.`);
         prevManifest.splice(prevMatchIndex, 1);
@@ -348,18 +380,7 @@ export function addImageToPrev(
     }
 
     onBeforeManifestPush();
-    prevManifest.push({
-        fileName: firmwareFileName,
-        fileVersion: parsedImage.fileVersion,
-        fileSize: parsedImage.totalImageSize,
-        originalUrl,
-        url: getRepoFirmwareFileUrl(manufacturer, firmwareFileName, PREV_IMAGES_DIR),
-        imageType: parsedImage.imageType,
-        manufacturerCode: parsedImage.manufacturerCode,
-        sha512: computeSHA512(firmwareBuffer),
-        otaHeaderString: parsedImage.otaHeaderString.replaceAll("\u0000", ""),
-        ...extraMetas,
-    });
+    prevManifest.push(newMetas);
 }
 
 export function addImageToBase(
@@ -379,6 +400,23 @@ export function addImageToBase(
     extraMetas: ExtraMetas,
     onBeforeManifestPush: () => void,
 ): void {
+    const newMetas: RepoImageMeta = {
+        fileName: firmwareFileName,
+        fileVersion: parsedImage.fileVersion,
+        fileSize: parsedImage.totalImageSize,
+        originalUrl,
+        url: getRepoFirmwareFileUrl(manufacturer, firmwareFileName, BASE_IMAGES_DIR),
+        imageType: parsedImage.imageType,
+        manufacturerCode: parsedImage.manufacturerCode,
+        sha512: computeSHA512(firmwareBuffer),
+        otaHeaderString: parsedImage.otaHeaderString.replaceAll("\u0000", ""),
+        ...extraMetas,
+    };
+
+    if (hasManufacturerImage(baseManifest, newMetas, extraMetas.manufacturerName)) {
+        throw new Error("Image already present for manufacturer");
+    }
+
     if (isNewer) {
         console.log(`${logPrefix} Base manifest has older version ${baseMatch.fileVersion}. Replacing with ${parsedImage.fileVersion}.`);
 
@@ -421,16 +459,5 @@ export function addImageToBase(
     }
 
     onBeforeManifestPush();
-    baseManifest.push({
-        fileName: firmwareFileName,
-        fileVersion: parsedImage.fileVersion,
-        fileSize: parsedImage.totalImageSize,
-        originalUrl,
-        url: getRepoFirmwareFileUrl(manufacturer, firmwareFileName, BASE_IMAGES_DIR),
-        imageType: parsedImage.imageType,
-        manufacturerCode: parsedImage.manufacturerCode,
-        sha512: computeSHA512(firmwareBuffer),
-        otaHeaderString: parsedImage.otaHeaderString.replaceAll("\u0000", ""),
-        ...extraMetas,
-    });
+    baseManifest.push(newMetas);
 }

src/find_matches.ts

@@ -0,0 +1,46 @@
+import {BASE_INDEX_MANIFEST_FILENAME, PREV_INDEX_MANIFEST_FILENAME, readManifest} from "./common";
+import type {RepoImageMeta} from "./types";
+
+const USAGE = `Usage: tsx src/find_matches.ts <BASE|PREV> <imageType> <manufacturerCode> [modelId] [manufacturerName]
+       Examples:
+         - tsx src/find_matches.ts BASE 287 4107
+         - tsx src/find_matches.ts BASE 287 4107 "abcd" "efgh"
+`;
+
+function getImageMetas(
+    imageList: RepoImageMeta[],
+    imageType: number,
+    manufacturerCode: number,
+    modelId: string | undefined,
+    manufacturerName: string | undefined,
+): RepoImageMeta[] | undefined {
+    return imageList
+        .filter(
+            (i) =>
+                i.imageType === imageType &&
+                i.manufacturerCode === manufacturerCode &&
+                (!i.modelId || !modelId || i.modelId === modelId) &&
+                (!i.manufacturerName || !manufacturerName || i.manufacturerName.includes(manufacturerName)),
+        )
+        .sort((a, b) => a.fileVersion - b.fileVersion);
+}
+
+function main(): void {
+    const args = process.argv.slice(2);
+
+    if (args.length < 2) {
+        throw new Error(USAGE);
+    }
+
+    const manifestName = args[0] === "PREV" ? PREV_INDEX_MANIFEST_FILENAME : BASE_INDEX_MANIFEST_FILENAME;
+    const imageType = Number(args[1]);
+    const manufacturerCode = Number(args[2]);
+    const modelId = args[3];
+    const manufacturerName = args[4] || undefined;
+    const manifest = readManifest(manifestName);
+    const matches = getImageMetas(manifest, imageType, manufacturerCode, modelId, manufacturerName);
+
+    console.log(matches);
+}
+
+main();

src/ghw_concat_cacerts.ts

@@ -8,7 +8,7 @@ export const CACERTS_DIR = "cacerts";
 export const CACERTS_CONCAT_FILEPATH = "cacerts.pem";
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-export function concatCaCerts(github: Octokit, core: typeof CoreApi, context: Context): void {
+export function concatCaCerts(_github: Octokit, core: typeof CoreApi, _context: Context): void {
     let pemContents = "";
 
     for (const pem of readdirSync(CACERTS_DIR)) {

src/ghw_identify_stacks.ts

@@ -76,7 +76,7 @@ function parseSubElements(otaData: Buffer, totalImageSize: number): OtaSubElemen
     return elements;
 }
 
-export function identifyStacks(github: Octokit, core: typeof CoreApi, context: Context): void {
+export function identifyStacks(_github: Octokit, core: typeof CoreApi, _context: Context): void {
     try {
         const firmwareList: FirmwareStack[] = [];
         const baseManifest = readManifest(BASE_INDEX_MANIFEST_FILENAME);

src/ghw_overwrite_cache.ts

@@ -5,7 +5,7 @@ import type {Octokit} from "@octokit/rest";
 
 import {ALL_AUTODL_MANUFACTURERS, CACHE_DIR} from "./common.js";
 
-export async function overwriteCache(github: Octokit, core: typeof CoreApi, context: Context, manufacturersCSV?: string): Promise<void> {
+export async function overwriteCache(_github: Octokit, core: typeof CoreApi, _context: Context, manufacturersCSV?: string): Promise<void> {
     if (!existsSync(CACHE_DIR)) {
         mkdirSync(CACHE_DIR, {recursive: true});
     }

src/ghw_reprocess_all_images.ts

@@ -61,9 +61,9 @@ function get3rdPartyDir(meta: RepoImageMeta): string | undefined {
 }
 
 async function download3rdParties(
-    github: Octokit,
+    _github: Octokit,
     core: typeof CoreApi,
-    context: Context,
+    _context: Context,
     /* v8 ignore next */ outDirFinder = get3rdPartyDir,
 ): Promise<void> {
     if (!process.env.NODE_EXTRA_CA_CERTS) {
@@ -213,7 +213,7 @@ async function download3rdParties(
     core.info(`Prev manifest now contains ${prevManifest.length} images.`);
 }
 
-function checkImagesAgainstManifests(github: Octokit, core: typeof CoreApi, context: Context, removeNotInManifest: boolean): void {
+function checkImagesAgainstManifests(_github: Octokit, core: typeof CoreApi, _context: Context, removeNotInManifest: boolean): void {
     for (const [manifestName, imagesDir, moveDir] of [
         [PREV_INDEX_MANIFEST_FILENAME, PREV_IMAGES_DIR, NOT_IN_PREV_MANIFEST_IMAGES_DIR],
         [BASE_INDEX_MANIFEST_FILENAME, BASE_IMAGES_DIR, NOT_IN_BASE_MANIFEST_IMAGES_DIR],

src/ghw_run_autodl.ts

@@ -5,7 +5,7 @@ import type {Octokit} from "@octokit/rest";
 
 import {ALL_AUTODL_MANUFACTURERS, BASE_INDEX_MANIFEST_FILENAME, CACHE_DIR, PREV_INDEX_MANIFEST_FILENAME, TMP_DIR, writeManifest} from "./common.js";
 
-export async function runAutodl(github: Octokit, core: typeof CoreApi, context: Context, manufacturersCSV?: string): Promise<void> {
+export async function runAutodl(_github: Octokit, core: typeof CoreApi, _context: Context, manufacturersCSV?: string): Promise<void> {
     const manufacturers = manufacturersCSV ? manufacturersCSV.trim().split(",") : ALL_AUTODL_MANUFACTURERS;
 
     core.info("Setup...");

src/z2m_find_match.ts

@@ -0,0 +1,72 @@
+/**
+ * Helper script to quickly retrieve an image meta in the same manner as Zigbee2MQTT.
+ */
+import {BASE_INDEX_MANIFEST_FILENAME, PREV_INDEX_MANIFEST_FILENAME, readManifest} from "./common.js";
+import type {ImageHeader, RepoImageMeta} from "./types.js";
+
+const USAGE = `Usage: tsx src/z2m_find_match.ts <BASE|PREV> <current> [device] [extraMetas]
+       Example: tsx src/z2m_find_match.ts BASE '{"imageType": 287, "manufacturerCode": 4107, "fileVersion": 16786436}' '{"modelID": "", "manufacturerName": "Philips"}'
+       Formats:
+         current: '{"imageType": number, "fileVersion": number, "manufacturerCode": number, "hardwareVersion": number | undefined}'
+         device [optional]: '{"modelID": string, "manufacturerName": string}'
+         extraMetas [optional]: '{"modelId": string, "otaHeaderString": string, "hardwareVersionMin": number | undefined, "hardwareVersionMax": number | undefined, "manufacturerName": string | undefined}'
+`;
+
+// #region Z2M
+interface ImageInfo {
+    imageType: ImageHeader["imageType"];
+    fileVersion: ImageHeader["fileVersion"];
+    manufacturerCode: ImageHeader["manufacturerCode"];
+    hardwareVersion?: number;
+}
+
+interface Device {
+    modelID: string;
+    manufacturerName: string;
+}
+
+type ExtraMetas = Pick<RepoImageMeta, "modelId" | "otaHeaderString" | "hardwareVersionMin" | "hardwareVersionMax"> & {
+    manufacturerName?: string;
+};
+
+function getImageMeta(imageList: RepoImageMeta[], current: ImageInfo, device: Device, extraMetas: ExtraMetas): RepoImageMeta | undefined {
+    return imageList.find(
+        (i) =>
+            i.imageType === current.imageType &&
+            i.manufacturerCode === current.manufacturerCode &&
+            (i.minFileVersion === undefined || current.fileVersion >= i.minFileVersion) &&
+            (i.maxFileVersion === undefined || current.fileVersion <= i.maxFileVersion) &&
+            // let extra metas override the match from device.modelID, same for manufacturerName
+            (!i.modelId || i.modelId === device.modelID || i.modelId === extraMetas.modelId) &&
+            (!i.manufacturerName ||
+                i.manufacturerName.includes(device.manufacturerName!) ||
+                i.manufacturerName.includes(extraMetas.manufacturerName!)) &&
+            (!extraMetas.otaHeaderString || i.otaHeaderString === extraMetas.otaHeaderString) &&
+            (i.hardwareVersionMin === undefined ||
+                (current.hardwareVersion !== undefined && current.hardwareVersion >= i.hardwareVersionMin) ||
+                (extraMetas.hardwareVersionMin !== undefined && extraMetas.hardwareVersionMin >= i.hardwareVersionMin)) &&
+            (i.hardwareVersionMax === undefined ||
+                (current.hardwareVersion !== undefined && current.hardwareVersion <= i.hardwareVersionMax) ||
+                (extraMetas.hardwareVersionMax !== undefined && extraMetas.hardwareVersionMax <= i.hardwareVersionMax)),
+    );
+}
+// #endregion Z2M
+
+function main(): void {
+    const args = process.argv.slice(2);
+
+    if (args.length < 2) {
+        throw new Error(USAGE);
+    }
+
+    const manifestName = args[0] === "PREV" ? PREV_INDEX_MANIFEST_FILENAME : BASE_INDEX_MANIFEST_FILENAME;
+    const current = JSON.parse(args[1]);
+    const device = args[2] ? JSON.parse(args[2]) : {};
+    const extraMetas = args[3] ? JSON.parse(args[3]) : {};
+    const manifest = readManifest(manifestName);
+    const match = getImageMeta(manifest, current, device, extraMetas);
+
+    console.log(match);
+}
+
+main();