Support Separate Targets Repo

graphql.schema.json

@@ -1055,6 +1055,18 @@
  "description": null,
  "fields": null,
  "inputFields": [
+ {
+ "name": "hardwareArtifactUrl",
+ "description": null,
+ "type": {
+ "kind": "SCALAR",
+ "name": "String",
+ "ofType": null
+ },
+ "defaultValue": null,
+ "isDeprecated": false,
+ "deprecationReason": null
+ },
  {
  "name": "owner",
  "description": null,

src/api/index.ts

@@ -138,10 +138,16 @@ export default class ApiServer {
  targetsLoader
  );
 
+ const targetStorageGitPath = path.join(
+ config.userDataPath,
+ 'firmwares',
+ 'binary-targets'
+ );
+
  const deviceDescriptionsLoader = new DeviceDescriptionsLoader(
  logger,
  config.PATH,
- path.join(config.userDataPath, 'firmwares', 'binary-targets'),
+ targetStorageGitPath,
  path.join(config.userDataPath, 'firmwares', 'device-options')
  );
  const cloudBinariesCache = new CloudBinariesCache(
@@ -158,6 +164,7 @@ export default class ApiServer {
  firmwareBuilder,
  deviceDescriptionsLoader,
  cloudBinariesCache,
+ targetStorageGitPath,
  logger
  );
 

src/api/src/graphql/inputs/GitRepositoryInput.ts

@@ -17,17 +17,22 @@ export default class GitRepository {
  @Field()
  srcFolder: string;
 
+ @Field({ nullable: true })
+ hardwareArtifactUrl: string | null;
+
  constructor(
  url: string,
  owner: string,
  repositoryName: string,
  rawRepoUrl: string,
- srcFolder: string
+ srcFolder: string,
+ hardwareArtifactUrl: string | null
  ) {
  this.url = url;
  this.owner = owner;
  this.repositoryName = repositoryName;
  this.rawRepoUrl = rawRepoUrl;
  this.srcFolder = srcFolder;
+ this.hardwareArtifactUrl = hardwareArtifactUrl;
  }
 }

src/api/src/library/AmazonS3/index.ts

@@ -0,0 +1,29 @@
+import fetch, { Headers } from 'node-fetch';
+import { writeFile } from 'fs/promises';
+import fs from 'fs';
+
+export default class AmazonS3 {
+ async downloadIfModified(url: string, outputFile: string): Promise<boolean> {
+ const headers = new Headers();
+ if (fs.existsSync(outputFile)) {
+ const stat = await fs.promises.stat(outputFile);
+ headers.append('If-Modified-Since', stat.mtime.toUTCString());
+ }
+
+ const response = await fetch(url, { headers });
+
+ if (response.status === 304) {
+ return false;
+ }
+
+ if (response.status === 200) {
+ const buffer = await response.buffer();
+ await writeFile(outputFile, buffer);
+ return true;
+ }
+
+ throw new Error(
+ `Error encountered while retrieving ${url}, ${response.status} - ${response.statusText}`
+ );
+ }
+}

src/api/src/services/BinaryFlashingStrategy/BinaryConfigurator/index.ts

@@ -36,12 +36,18 @@ export default class BinaryConfigurator {
  buildBinaryConfigFlags(
  outputDirectory: string,
  firmwareBinaryPath: string,
- hardwareDefinitionsPath: string,
+ hardwareDefinitionsPath: string | null,
+ firmwareArtifactsDirPath: string,
  params: BuildFlashFirmwareParams
  ): string[][] {
  const flags: string[][] = [];
 
- flags.push(['--dir', hardwareDefinitionsPath]);
+ if (hardwareDefinitionsPath && firmwareArtifactsDirPath) {
+ flags.push(['--dir', hardwareDefinitionsPath]);
+ flags.push(['--fdir', firmwareArtifactsDirPath]);
+ } else {
+ flags.push(['--dir', firmwareArtifactsDirPath]);
+ }
 
  const [manufacturer, subType, device, uploadMethod] =
  params.target.split('.');

src/api/src/services/BinaryFlashingStrategy/CloudBinariesCache/index.ts

@@ -1,9 +1,8 @@
-import fetch from 'node-fetch';
-import { writeFile } from 'fs/promises';
 import extractZip from 'extract-zip';
 import path from 'path';
 import mkdirp from 'mkdirp';
-import fs from 'fs';
+import { removeDirectoryContents } from '../../FlashingStrategyLocator/artefacts';
+import AmazonS3 from '../../../library/AmazonS3';
 
 export default class CloudBinariesCache {
  constructor(private baseURL: string, private firmwareCachePath: string) {}
@@ -17,28 +16,19 @@ export default class CloudBinariesCache {
  await mkdirp(workingDir);
  const firmwareCacheDir = path.join(workingDir, 'firmware');
 
- // if we have firmware in our local cache already no need to download it the second time.
- if (fs.existsSync(firmwareCacheDir)) {
- return firmwareCacheDir;
- }
-
  const cacheUrl = `${this.baseURL}/${repositoryName}/${commitHash}/firmware.zip`;
- const response = await fetch(cacheUrl);
- console.log(response);
- if (response.status !== 200) {
- throw new Error(
- `cached build for ${repositoryName}/${commitHash} is not available`
- );
- }
-
  const outputZipFile = path.join(workingDir, 'firmware.zip');
- const buffer = await response.buffer();
- await writeFile(outputZipFile, buffer);
 
- await extractZip(outputZipFile, {
- dir: workingDir,
- });
+ if (await new AmazonS3().downloadIfModified(cacheUrl, outputZipFile)) {
+ await extractZip(outputZipFile, {
+ dir: workingDir,
+ });
+ }
 
  return firmwareCacheDir;
  }
+
+ async clearCache() {
+ await removeDirectoryContents(this.firmwareCachePath);
+ }
 }

src/api/src/services/BinaryFlashingStrategy/DeviceDescriptionsLoader/index.ts

@@ -1,5 +1,8 @@
+import extractZip from 'extract-zip';
 import { Service } from 'typedi';
 import path from 'path';
+import mkdirp from 'mkdirp';
+import semver from 'semver';
 import FirmwareSource from '../../../models/enum/FirmwareSource';
 import TargetArgs from '../../../graphql/args/Target';
 import { LoggerService } from '../../../logger';
@@ -19,10 +22,12 @@ import TargetUserDefinesFactory from '../../../factories/TargetUserDefinesFactor
 import UserDefineKey from '../../../library/FirmwareBuilder/Enum/UserDefineKey';
 import PullRequest from '../../../models/PullRequest';
 import { removeDirectoryContents } from '../../FlashingStrategyLocator/artefacts';
+import AmazonS3 from '../../../library/AmazonS3';
 
 export interface GitRepository {
  url: string;
  srcFolder: string;
+ hardwareArtifactUrl: string | null;
 }
 
 export interface FirmwareVersion {
@@ -125,7 +130,17 @@ export default class DeviceDescriptionsLoader {
  );
  const devices: Device[] = [];
  Object.keys(data).forEach((id) => {
- devices.push(this.configToDevice(id, data[id].category, data[id].config));
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ const { min_version } = data[id].config;
+ if (
+ !min_version ||
+ args.source !== FirmwareSource.GitTag ||
+ semver.gte(args.gitTag, min_version)
+ ) {
+ devices.push(
+ this.configToDevice(id, data[id].category, data[id].config)
+ );
+ }
  });
  return devices;
  }
@@ -149,6 +164,25 @@ export default class DeviceDescriptionsLoader {
  gitPath,
  });
 
+ if (gitRepository.hardwareArtifactUrl) {
+ const workingDir = path.join(gitRepositoryPath, 'hardware');
+ await mkdirp(workingDir);
+ const outputZipFile = path.join(workingDir, 'hardware.zip');
+
+ if (
+ await new AmazonS3().downloadIfModified(
+ gitRepository.hardwareArtifactUrl,
+ outputZipFile
+ )
+ ) {
+ await extractZip(outputZipFile, {
+ dir: workingDir,
+ });
+ }
+
+ return workingDir;
+ }
+
  const firmwareDownload = new GitFirmwareDownloader(
  {
  baseDirectory: gitRepositoryPath,
@@ -339,14 +373,14 @@ export default class DeviceDescriptionsLoader {
  async clearCache() {
  await this.targetsMutex.tryLockWithTimeout(60000);
  try {
- return await removeDirectoryContents(this.targetStorageGitPath);
+ await removeDirectoryContents(this.targetStorageGitPath);
  } finally {
  this.targetsMutex.unlock();
  }
 
  await this.deviceOptionsMutex.tryLockWithTimeout(60000);
  try {
- return await removeDirectoryContents(this.deviceOptionsGitPath);
+ await removeDirectoryContents(this.deviceOptionsGitPath);
  } finally {
  this.deviceOptionsMutex.unlock();
  }

src/api/src/services/BinaryFlashingStrategy/TargetsJSONLoader/index.ts

@@ -15,6 +15,7 @@ export interface DeviceDescription {
  overlay?: { [key: string]: string };
  frequency?: number;
  type?: string;
+ min_version?: string;
 }
 
 export type TargetsJSONRaw = {

src/api/src/services/BinaryFlashingStrategy/index.ts

@@ -62,6 +62,7 @@ export default class BinaryFlashingStrategyService implements FlashingStrategy {
  private builder: FirmwareBuilder,
  private deviceDescriptionsLoader: DeviceDescriptionsLoader,
  private cloudBinariesCache: CloudBinariesCache,
+ private targetStorageGitPath: string,
  private logger: LoggerService
  ) {
  this.mutex = new Mutex();
@@ -473,6 +474,7 @@ export default class BinaryFlashingStrategyService implements FlashingStrategy {
  {
  url: gitRepositoryUrl,
  srcFolder: gitRepositorySrcFolder,
+ hardwareArtifactUrl: gitRepository.hardwareArtifactUrl,
  }
  );
 
@@ -481,7 +483,7 @@ export default class BinaryFlashingStrategyService implements FlashingStrategy {
  const workingDirectory = await this.createWorkingDirectory(params.target);
  const outputDirectory = await this.createWorkingDirectory(params.target);
  let firmwareBinFile = '';
- let hardwareDescriptionsPath = firmwareSourcePath;
+ let firmwareDescriptionsPath = firmwareSourcePath;
  let flasherPath = path.join(
  firmwareSourcePath,
  'python',
@@ -510,13 +512,13 @@ export default class BinaryFlashingStrategyService implements FlashingStrategy {
  if (fs.existsSync(flasherPyzPath)) {
  flasherPath = flasherPyzPath;
  }
- hardwareDescriptionsPath = cacheLocation;
+ firmwareDescriptionsPath = cacheLocation;
  this.logger.log('paths', {
  cacheLocation,
  cachedBinaryPath,
  flasherPyzPath,
  sourceFirmwareBinaryPath: sourceFirmwareBinPath,
- hardwareDescriptionsPath,
+ hardwareDescriptionsPath: firmwareDescriptionsPath,
  });
  } catch (e) {
  this.logger.log(
@@ -530,7 +532,7 @@ export default class BinaryFlashingStrategyService implements FlashingStrategy {
  }
 
  // we were not able to find cloud binaries, so we will build them on the spot
- if (hardwareDescriptionsPath === firmwareSourcePath) {
+ if (firmwareDescriptionsPath === firmwareSourcePath) {
  const target = this.getCompileTarget(config);
  sourceFirmwareBinPath = await this.compileBinary(
  target,
@@ -557,13 +559,24 @@ export default class BinaryFlashingStrategyService implements FlashingStrategy {
  BuildFirmwareStep.BUILDING_FIRMWARE
  );
 
- const flasherArgs: string[][] =
- this.binaryConfigurator.buildBinaryConfigFlags(
+ let flasherArgs: string[][];
+ if (gitRepository.hardwareArtifactUrl) {
+ flasherArgs = this.binaryConfigurator.buildBinaryConfigFlags(
  outputDirectory,
  firmwareBinFile,
- hardwareDescriptionsPath,
+ this.targetStorageGitPath,
+ firmwareDescriptionsPath,
  params
  );
+ } else {
+ flasherArgs = this.binaryConfigurator.buildBinaryConfigFlags(
+ outputDirectory,
+ firmwareBinFile,
+ null,
+ firmwareDescriptionsPath,
+ params
+ );
+ }
  const binaryConfiguratorResult = await this.binaryConfigurator.run(
  flasherPath,
  flasherArgs,
@@ -629,10 +642,11 @@ export default class BinaryFlashingStrategyService implements FlashingStrategy {
 
  async clearFirmwareFiles(): Promise<void> {
  await this.deviceDescriptionsLoader.clearCache();
+ await this.cloudBinariesCache.clearCache();
 
  this.logger.log('BinaryConfigurator - clearFirmwareFiles', {
  firmwaresPath: this.firmwaresPath,
  });
- return removeDirectoryContents(this.firmwaresPath);
+ await removeDirectoryContents(this.firmwaresPath);
  }
 }

src/api/src/services/FlashingStrategyLocator/artefacts.ts

@@ -110,7 +110,7 @@ export const removeDirectoryContents = async (firmwaresPath: string) => {
  return;
  }
  const files = await listFiles(firmwaresPath);
- if (files.length > 4) {
+ if (files.length > 7) {
  throw new Error(`unexpected number of files to remove: ${files}`);
  }
  await Promise.all(files.map((item) => rmrf(item)));

src/api/src/services/PlatformioFlashingStrategy/index.ts

@@ -424,6 +424,7 @@ export default class PlatformioFlashingStrategyService
  this.logger.log('PlatformioConfigurator - clearFirmwareFiles', {
  firmwaresPath: this.firmwaresPath,
  });
+ await this.userDefinesBuilder.clearFiles();
  return removeDirectoryContents(this.firmwaresPath);
  }
 }

src/api/src/services/UserDefinesBuilder/index.ts

@@ -58,4 +58,8 @@ export default class UserDefinesBuilder {
  return compatibleKeys.indexOf(userDefine.key) > -1;
  });
  }
+
+ async clearFiles() {
+ await this.userDefinesLoader.clearFiles();
+ }
 }