add changeset

Signed-off-by: Fredrik Adelöw <freben@gmail.com>

Author: freben

.changeset/cuddly-flowers-film.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Avoid duplicate work by comparing previous processing rounds with the next

plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts

@@ -76,6 +76,7 @@ describe('DefaultCatalogProcessingEngine', () => {
               kind: 'Location',
               metadata: { name: 'test' },
             },
+            hash: '',
             state: new Map(),
             nextUpdateAt: '',
             lastDiscoveryAt: '',
@@ -137,6 +138,7 @@ describe('DefaultCatalogProcessingEngine', () => {
               kind: 'Location',
               metadata: { name: 'test' },
             },
+            hash: '',
             state: new Map(),
             nextUpdateAt: '',
             lastDiscoveryAt: '',

plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.ts

@@ -162,10 +162,11 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine {
 
           const hash = hashBuilder.digest('hex');
           if (hash === previousHash) {
-            console.log('skipping ', entityRef);
+            // If nothing changed in our produced outputs, we cannot have any
+            // significant effect on our surroundings; therefore, we just abort
+            // without any updates / stitching.
             return;
           }
-          console.log('going ahead with ', entityRef);
 
           // If the result was marked as not OK, it signals that some part of the
           // processing pipeline threw an exception. This can happen both as part of

plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts

@@ -124,11 +124,12 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
     options: UpdateProcessedEntityOptions,
   ): Promise<void> {
     const tx = txOpaque as Knex.Transaction;
-    const { id, errors } = options;
+    const { id, errors, hash } = options;
 
     await tx<DbRefreshStateRow>('refresh_state')
       .update({
         errors,
+        hash,
       })
       .where('entity_id', id);
   }
@@ -494,7 +495,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
             processedEntity: i.processed_entity
               ? (JSON.parse(i.processed_entity) as Entity)
               : undefined,
-            hash: i.hash,
+            hash: i.hash || '',
             nextUpdateAt: i.next_update_at,
             lastDiscoveryAt: i.last_discovery_at,
             state: i.cache