Split workflow processes. (#588)

* Split workflow processes.

* fix

src/common.ts

@@ -19,6 +19,13 @@ export const BASE_INDEX_MANIFEST_FILENAME = 'index.json';
 export const PREV_INDEX_MANIFEST_FILENAME = 'index1.json';
 export const CACHE_DIR = '.cache';
 export const TMP_DIR = 'tmp';
+export const PR_ARTIFACT_DIR = 'pr';
+export const PR_DIFF_FILENAME = 'PR_DIFF';
+export const PR_ERROR_FILENAME = 'PR_ERROR';
+export const PR_NUMBER_FILENAME = 'PR_NUMBER';
+export const PR_ARTIFACT_DIFF_FILEPATH = path.join(PR_ARTIFACT_DIR, PR_DIFF_FILENAME);
+export const PR_ARTIFACT_ERROR_FILEPATH = path.join(PR_ARTIFACT_DIR, PR_ERROR_FILENAME);
+export const PR_ARTIFACT_NUMBER_FILEPATH = path.join(PR_ARTIFACT_DIR, PR_NUMBER_FILENAME);
 /**
  * 'ikea_new' first, to prioritize downloads from new URL
  */

src/ghw_check_ota_pr.ts

@@ -0,0 +1,68 @@
+import type CoreApi from '@actions/core';
+import type {Context} from '@actions/github/lib/context';
+import type {Octokit} from '@octokit/rest';
+
+import assert from 'assert';
+import {existsSync, mkdirSync, writeFileSync} from 'fs';
+
+import {
+    BASE_INDEX_MANIFEST_FILENAME,
+    execute,
+    PR_ARTIFACT_DIFF_FILEPATH,
+    PR_ARTIFACT_DIR,
+    PR_ARTIFACT_ERROR_FILEPATH,
+    PR_ARTIFACT_NUMBER_FILEPATH,
+    PREV_INDEX_MANIFEST_FILENAME,
+    readManifest,
+    writeManifest,
+} from './common.js';
+import {getChangedOtaFiles} from './ghw_get_changed_ota_files.js';
+import {processOtaFiles} from './ghw_process_ota_files.js';
+
+function throwError(comment: string): void {
+    writeFileSync(PR_ARTIFACT_ERROR_FILEPATH, comment);
+
+    throw new Error(comment);
+}
+
+export async function checkOtaPR(github: Octokit, core: typeof CoreApi, context: Context): Promise<void> {
+    assert(context.payload.pull_request, 'Not a pull request');
+    assert(!context.payload.pull_request.merged, 'Should not be executed on a merged pull request');
+
+    if (!existsSync(PR_ARTIFACT_DIR)) {
+        mkdirSync(PR_ARTIFACT_DIR, {recursive: true});
+    }
+
+    writeFileSync(PR_ARTIFACT_NUMBER_FILEPATH, context.issue.number.toString(10), 'utf8');
+
+    const baseManifest = readManifest(BASE_INDEX_MANIFEST_FILENAME);
+    const prevManifest = readManifest(PREV_INDEX_MANIFEST_FILENAME);
+
+    try {
+        const filePaths = await getChangedOtaFiles(
+            github,
+            core,
+            context,
+            `${context.payload.pull_request.base.sha}...${context.payload.pull_request.head.sha}`,
+            true,
+        );
+
+        await processOtaFiles(github, core, context, filePaths, baseManifest, prevManifest);
+    } catch (error) {
+        throwError((error as Error).message);
+    }
+
+    writeManifest(PREV_INDEX_MANIFEST_FILENAME, prevManifest);
+    writeManifest(BASE_INDEX_MANIFEST_FILENAME, baseManifest);
+
+    core.info(`Prev manifest has ${prevManifest.length} images.`);
+    core.info(`Base manifest has ${baseManifest.length} images.`);
+
+    const diff = await execute(`git diff`);
+
+    core.startGroup('diff');
+    core.info(diff);
+    core.endGroup();
+
+    writeFileSync(PR_ARTIFACT_DIFF_FILEPATH, diff);
+}

src/ghw_create_autodl_release.ts

@@ -2,8 +2,9 @@ import type CoreApi from '@actions/core';
 import type {Context} from '@actions/github/lib/context';
 import type {Octokit} from '@octokit/rest';
 
+import type {RepoImageMeta} from './types.js';
+
 import {BASE_IMAGES_DIR, BASE_INDEX_MANIFEST_FILENAME, execute, PREV_IMAGES_DIR, PREV_INDEX_MANIFEST_FILENAME, readManifest} from './common.js';
-import {RepoImageMeta} from './types.js';
 
 // about 3 lines
 const MAX_RELEASE_NOTES_LENGTH = 380;

src/ghw_get_changed_ota_files.ts

@@ -0,0 +1,34 @@
+import type CoreApi from '@actions/core';
+import type {Context} from '@actions/github/lib/context';
+import type {Octokit} from '@octokit/rest';
+
+import assert from 'assert';
+
+import {BASE_IMAGES_DIR} from './common.js';
+
+export async function getChangedOtaFiles(
+    github: Octokit,
+    core: typeof CoreApi,
+    context: Context,
+    basehead: string,
+    isPR: boolean,
+): Promise<string[]> {
+    // NOTE: includes up to 300 files, per https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#compare-two-commits
+    const compare = await github.rest.repos.compareCommitsWithBasehead({
+        owner: context.repo.owner,
+        repo: context.repo.repo,
+        basehead,
+    });
+
+    assert(compare.data.files && compare.data.files.length > 0, 'No file');
+
+    core.info(`Changed files: ${compare.data.files.map((f) => f.filename).join(', ')}`);
+
+    const fileList = compare.data.files.filter((f) => f.filename.startsWith(`${BASE_IMAGES_DIR}/`));
+
+    if (isPR && fileList.length !== compare.data.files.length) {
+        throw new Error(`Detected changes in files outside of \`images\` directory. This is not allowed for a pull request with OTA files.`);
+    }
+
+    return fileList.map((f) => f.filename);
+}

src/ghw_process_ota_files.ts

@@ -2,9 +2,8 @@ import type CoreApi from '@actions/core';
 import type {Context} from '@actions/github/lib/context';
 import type {Octokit} from '@octokit/rest';
 
-import type {ExtraMetas, GHExtraMetas} from './types';
+import type {ExtraMetas, GHExtraMetas, RepoImageMeta} from './types.js';
 
-import assert from 'assert';
 import {readFileSync, renameSync} from 'fs';
 import path from 'path';
 
@@ -12,8 +11,6 @@ import {
     addImageToBase,
     addImageToPrev,
     BASE_IMAGES_DIR,
-    BASE_INDEX_MANIFEST_FILENAME,
-    execute,
     findMatchImage,
     getOutDir,
     getParsedImageStatus,
@@ -21,10 +18,7 @@ import {
     ParsedImageStatus,
     parseImageHeader,
     PREV_IMAGES_DIR,
-    PREV_INDEX_MANIFEST_FILENAME,
-    readManifest,
     UPGRADE_FILE_IDENTIFIER,
-    writeManifest,
 } from './common.js';
 
 const EXTRA_METAS_PR_BODY_START_TAG = '```json';
@@ -75,97 +69,46 @@ async function parsePRBodyExtraMetas(github: Octokit, core: typeof CoreApi, cont
                 }
             }
         } catch (error) {
-            const failureComment = `Invalid extra metas in pull request body: ` + (error as Error).message;
-
-            core.error(failureComment);
-
-            await github.rest.issues.createComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: context.issue.number,
-                body: failureComment,
-            });
-
-            throw new Error(failureComment);
+            throw new Error(`Invalid extra metas in pull request body: ${(error as Error).message}`);
         }
     }
 
     return extraMetas;
 }
 
-export async function updateOtaPR(github: Octokit, core: typeof CoreApi, context: Context, fileParam: string): Promise<void> {
-    assert(fileParam, 'No file found in pull request.');
-    assert(context.payload.pull_request, 'Not a pull request');
-
-    const fileParamArr = fileParam.trim().split(',');
-    // take care of empty strings (GH workflow adds a comma at end), ignore files not stored in images dir
-    const fileList = fileParamArr.filter((f) => f.startsWith(`${BASE_IMAGES_DIR}/`));
-
-    assert(fileList.length > 0, 'No image found in pull request.');
-    core.info(`Images in pull request: ${fileList}.`);
-
-    const fileListWrongDir = fileParamArr.filter((f) => f.startsWith(`${PREV_IMAGES_DIR}/`));
-
-    if (fileListWrongDir.length > 0) {
-        const failureComment = `Detected files in 'images1':
-\`\`\`
-${fileListWrongDir.join('\n')}
-\`\`\`
-Please move all files to 'images' (in appropriate subfolders). The pull request will automatically determine the proper location on merge.`;
-
-        await github.rest.issues.createComment({
-            owner: context.repo.owner,
-            repo: context.repo.repo,
-            issue_number: context.issue.number,
-            body: failureComment,
-        });
-
-        throw new Error(failureComment);
-    }
-
-    const fileListNoIndex = fileParamArr.filter((f) => f.startsWith(BASE_INDEX_MANIFEST_FILENAME) || f.startsWith(PREV_INDEX_MANIFEST_FILENAME));
-
-    if (fileListNoIndex.length > 0) {
-        const failureComment = `Detected manual changes in ${fileListNoIndex.join(', ')}. Please remove these changes. The pull request will automatically determine the manifests on merge.`;
-
-        await github.rest.issues.createComment({
-            owner: context.repo.owner,
-            repo: context.repo.repo,
-            issue_number: context.issue.number,
-            body: failureComment,
-        });
-
-        throw new Error(failureComment);
-    }
-
-    // called at the top, fail early if invalid PR body metas
+export async function processOtaFiles(
+    github: Octokit,
+    core: typeof CoreApi,
+    context: Context,
+    filePaths: string[],
+    baseManifest: RepoImageMeta[],
+    prevManifest: RepoImageMeta[],
+): Promise<void> {
     const extraMetas = await parsePRBodyExtraMetas(github, core, context);
-    const baseManifest = readManifest(BASE_INDEX_MANIFEST_FILENAME);
-    const prevManifest = readManifest(PREV_INDEX_MANIFEST_FILENAME);
 
-    for (const file of fileList) {
-        core.startGroup(file);
-        core.info(`Processing '${file}'...`);
+    for (const filePath of filePaths) {
+        core.startGroup(filePath);
 
+        const logPrefix = `[${filePath}]`;
         let failureComment: string = '';
 
         try {
-            const firmwareFileName = path.basename(file);
-            const manufacturer = file.replace(BASE_IMAGES_DIR, '').replace(firmwareFileName, '').replaceAll('/', '').trim();
+            const firmwareFileName = path.basename(filePath);
+            const manufacturer = filePath.replace(BASE_IMAGES_DIR, '').replace(firmwareFileName, '').replaceAll('/', '').trim();
 
             if (!manufacturer) {
-                throw new Error(`\`${file}\` should be in its associated manufacturer subfolder.`);
+                throw new Error(`File should be in its associated manufacturer subfolder`);
             }
 
-            const firmwareBuffer = Buffer.from(readFileSync(file));
+            const firmwareBuffer = Buffer.from(readFileSync(filePath));
             const parsedImage = parseImageHeader(firmwareBuffer.subarray(firmwareBuffer.indexOf(UPGRADE_FILE_IDENTIFIER)));
 
-            core.info(`[${file}] Parsed image header:`);
+            core.info(`${logPrefix} Parsed image header:`);
             core.info(JSON.stringify(parsedImage, undefined, 2));
 
             const fileExtraMetas = getFileExtraMetas(extraMetas, firmwareFileName);
 
-            core.info(`[${file}] Extra metas:`);
+            core.info(`${logPrefix} Extra metas:`);
             core.info(JSON.stringify(fileExtraMetas, undefined, 2));
 
             const baseOutDir = getOutDir(manufacturer, BASE_IMAGES_DIR);
@@ -200,7 +143,7 @@ ${JSON.stringify(parsedImage, undefined, 2)}
                         case ParsedImageStatus.NEWER:
                         case ParsedImageStatus.NEW: {
                             addImageToPrev(
-                                `[${file}]`,
+                                logPrefix,
                                 statusToPrev === ParsedImageStatus.NEWER,
                                 prevManifest,
                                 prevMatchIndex,
@@ -214,7 +157,7 @@ ${JSON.stringify(parsedImage, undefined, 2)}
                                 fileExtraMetas,
                                 () => {
                                     // relocate file to prev
-                                    renameSync(file, file.replace(`${BASE_IMAGES_DIR}/`, `${PREV_IMAGES_DIR}/`));
+                                    renameSync(filePath, filePath.replace(`${BASE_IMAGES_DIR}/`, `${PREV_IMAGES_DIR}/`));
                                 },
                             );
 
@@ -240,7 +183,7 @@ ${JSON.stringify(parsedImage, undefined, 2)}
                 case ParsedImageStatus.NEWER:
                 case ParsedImageStatus.NEW: {
                     addImageToBase(
-                        `[${file}]`,
+                        logPrefix,
                         statusToBase === ParsedImageStatus.NEWER,
                         prevManifest,
                         prevOutDir,
@@ -267,41 +210,10 @@ ${JSON.stringify(parsedImage, undefined, 2)}
         }
 
         if (failureComment) {
-            core.error(`[${file}] ` + failureComment);
-            await github.rest.pulls.createReviewComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                pull_number: context.issue.number,
-                body: failureComment,
-                commit_id: context.payload.pull_request.head.sha,
-                path: file,
-                subject_type: 'file',
-            });
-
-            throw new Error(failureComment);
+            core.endGroup();
+            throw new Error(`${logPrefix} ${failureComment}`);
         }
 
         core.endGroup();
     }
-
-    writeManifest(PREV_INDEX_MANIFEST_FILENAME, prevManifest);
-    writeManifest(BASE_INDEX_MANIFEST_FILENAME, baseManifest);
-
-    core.info(`Prev manifest has ${prevManifest.length} images.`);
-    core.info(`Base manifest has ${baseManifest.length} images.`);
-
-    if (!context.payload.pull_request.merged) {
-        const diff = await execute(`git diff`);
-
-        await github.rest.issues.createComment({
-            owner: context.repo.owner,
-            repo: context.repo.repo,
-            issue_number: context.issue.number,
-            body: `Merging this pull request will add these changes in a following commit:
-\`\`\`diff
-${diff}
-\`\`\`
-`,
-        });
-    }
 }

src/ghw_report_ota_pr.ts

@@ -0,0 +1,82 @@
+import type CoreApi from '@actions/core';
+import type {Context} from '@actions/github/lib/context';
+import type {Octokit} from '@octokit/rest';
+
+import assert from 'assert';
+import {existsSync, readFileSync, writeFileSync} from 'fs';
+
+import {execute, PR_ARTIFACT_DIR, PR_DIFF_FILENAME, PR_ERROR_FILENAME, PR_NUMBER_FILENAME} from './common.js';
+
+export async function reportOtaPR(github: Octokit, core: typeof CoreApi, context: Context): Promise<void> {
+    assert(context.payload.workflow_run, 'Not a workflow run');
+
+    // XXX: context.payload.workflow_run is not typed...
+    const workflow_run = context.payload.workflow_run as Awaited<ReturnType<typeof github.rest.actions.getWorkflowRun>>['data'];
+
+    // workflow_run.conclusion: action_required, cancelled, failure, neutral, skipped, stale, success, timed_out, startup_failure, null
+    if (workflow_run.conclusion !== 'success' && workflow_run.conclusion !== 'failure') {
+        core.info(`Ignoring workflow run ${workflow_run.html_url} with conclusion ${workflow_run.conclusion}.`);
+
+        return;
+    }
+
+    const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+        owner: context.repo.owner,
+        repo: context.repo.repo,
+        run_id: workflow_run.id,
+    });
+    const matchArtifact = artifacts.data.artifacts.find((artifact) => artifact.name == PR_ARTIFACT_DIR);
+
+    assert(matchArtifact, `No artifact found for ${workflow_run.url}`);
+
+    const download = await github.rest.actions.downloadArtifact({
+        owner: context.repo.owner,
+        repo: context.repo.repo,
+        artifact_id: matchArtifact.id,
+        archive_format: 'zip',
+    });
+    const artifactZipFileName = `${PR_ARTIFACT_DIR}.zip`;
+
+    writeFileSync(artifactZipFileName, Buffer.from(download.data as ArrayBuffer));
+
+    const unzipOutput = await execute(`unzip ${artifactZipFileName}`);
+
+    core.info(unzipOutput);
+
+    assert(existsSync(PR_NUMBER_FILENAME), `Invalid artifact for ${workflow_run.html_url}`);
+
+    const prNumber = parseInt(readFileSync(PR_NUMBER_FILENAME, 'utf8'), 10);
+
+    core.info(`Running for pr#${prNumber} for ${workflow_run.html_url}`);
+
+    if (workflow_run.conclusion === 'failure') {
+        assert(existsSync(PR_ERROR_FILENAME), `Workflow failed but could not find ${PR_ERROR_FILENAME} for ${workflow_run.html_url}`);
+
+        const prError = readFileSync(PR_ERROR_FILENAME, 'utf8');
+
+        await github.rest.issues.createComment({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: prNumber,
+            body: prError,
+        });
+
+        throw new Error(prError);
+    } else if (workflow_run.conclusion === 'success') {
+        assert(existsSync(PR_DIFF_FILENAME), `Workflow succeeded but could not find ${PR_DIFF_FILENAME} for ${workflow_run.html_url}`);
+
+        const prDiff = readFileSync(PR_DIFF_FILENAME, 'utf8');
+
+        core.info(prDiff);
+        await github.rest.issues.createComment({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: prNumber,
+            body: `Merging this pull request will add these changes in a following commit:
+\`\`\`diff
+${prDiff}
+\`\`\`
+`,
+        });
+    }
+}

src/ghw_update_manifests.ts

@@ -0,0 +1,26 @@
+import type CoreApi from '@actions/core';
+import type {Context} from '@actions/github/lib/context';
+import type {Octokit} from '@octokit/rest';
+
+import assert from 'assert';
+
+import {BASE_INDEX_MANIFEST_FILENAME, PREV_INDEX_MANIFEST_FILENAME, readManifest, writeManifest} from './common.js';
+import {getChangedOtaFiles} from './ghw_get_changed_ota_files.js';
+import {processOtaFiles} from './ghw_process_ota_files.js';
+
+export async function updateManifests(github: Octokit, core: typeof CoreApi, context: Context): Promise<void> {
+    assert(context.eventName === 'push', 'Not a push');
+
+    const filePaths = await getChangedOtaFiles(github, core, context, `${context.payload.before}...${context.payload.after}`, false);
+    const baseManifest = readManifest(BASE_INDEX_MANIFEST_FILENAME);
+    const prevManifest = readManifest(PREV_INDEX_MANIFEST_FILENAME);
+
+    // will throw if anything goes wrong
+    await processOtaFiles(github, core, context, filePaths, baseManifest, prevManifest);
+
+    writeManifest(PREV_INDEX_MANIFEST_FILENAME, prevManifest);
+    writeManifest(BASE_INDEX_MANIFEST_FILENAME, baseManifest);
+
+    core.info(`Prev manifest has ${prevManifest.length} images.`);
+    core.info(`Base manifest has ${baseManifest.length} images.`);
+}

src/index.ts

@@ -1,9 +1,13 @@
 export * as common from './common.js';
+export {checkOtaPR} from './ghw_check_ota_pr.js';
 export {concatCaCerts} from './ghw_concat_cacerts.js';
 export {createAutodlRelease} from './ghw_create_autodl_release.js';
 export {createPRToDefault} from './ghw_create_pr_to_default.js';
+export {getChangedOtaFiles} from './ghw_get_changed_ota_files.js';
 export {overwriteCache} from './ghw_overwrite_cache.js';
+export {processOtaFiles} from './ghw_process_ota_files.js';
+export {reportOtaPR} from './ghw_report_ota_pr.js';
 export {reProcessAllImages} from './ghw_reprocess_all_images.js';
 export {runAutodl} from './ghw_run_autodl.js';
-export {updateOtaPR} from './ghw_update_ota_pr.js';
+export {updateManifests} from './ghw_update_manifests.js';
 export {processFirmwareImage} from './process_firmware_image.js';