Fix incremental table recreation when defaultProject not set (issue #2053)

Siby Abin Thomas · Siby Abin Thomas · commit 6a3b08bf387a · 2025-12-23T05:40:22.000+05:30
diff --git a/cli/api/commands/build.ts b/cli/api/commands/build.ts
@@ -49,8 +49,13 @@ export class Builder {
 
     const tableMetadataByTarget = new Map<string, dataform.ITableMetadata>();
 
+    // Normalize warehouse state targets to match compiled targets for consistent lookups.
+    // This is necessary because targets from the warehouse may have a database field set
+    // (from BigQuery credentials) while compiled targets may not have a database field
+    // if defaultDatabase is not specified in workflow_settings.yaml.
     this.warehouseState.tables.forEach(tableState => {
-      tableMetadataByTarget.set(targetStringifier.stringify(tableState.target), tableState);
+      const normalizedTarget = this.normalizeWarehouseTarget(tableState.target);
+      tableMetadataByTarget.set(targetStringifier.stringify(normalizedTarget), tableState);
     });
 
     const actions: dataform.IExecutionAction[] = [].concat(
@@ -121,4 +126,22 @@ export class Builder {
       actionDescriptor: action.actionDescriptor
     });
   }
+
+  private normalizeWarehouseTarget(warehouseTarget: dataform.ITarget): dataform.ITarget {
+    // If the warehouse target has a database field and the compiled graph doesn't have a
+    // defaultDatabase, we should remove the database field from the warehouse target
+    // to match the format of the compiled targets. This ensures consistent lookups
+    // when defaultDatabase is not specified in workflow_settings.yaml.
+    if (
+      warehouseTarget.database &&
+      !this.prunedGraph.projectConfig.defaultDatabase
+    ) {
+      return dataform.Target.create({
+        schema: warehouseTarget.schema,
+        name: warehouseTarget.name
+      });
+    }
+    // Otherwise return the warehouse target as-is
+    return warehouseTarget;
+  }
 }
diff --git a/core/actions/incremental_table_test.ts b/core/actions/incremental_table_test.ts
@@ -882,4 +882,39 @@ defaultIcebergConfig:
       });
     });
   });
+
+  test("incremental table without defaultProject in workflow_settings", () => {
+    const projectDir = tmpDirFixture.createNewTmpDir();
+    // Create workflow_settings without defaultProject (only defaultDataset and defaultLocation)
+    fs.writeFileSync(
+      path.join(projectDir, "workflow_settings.yaml"),
+      `defaultDataset: dataform
+defaultLocation: europe-west2
+`
+    );
+    fs.mkdirSync(path.join(projectDir, "definitions"));
+    fs.writeFileSync(
+      path.join(projectDir, "definitions/incremental_table_without_default_project.sqlx"),
+      `config {
+    type: "incremental",
+    name: "incremental_table_without_default_project"
+}
+
+select \${incremental()} as is_incremental`
+    );
+
+    const result = runMainInVm(coreExecutionRequestFromPath(projectDir));
+
+    expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]);
+    const compiledTable = result.compile.compiledGraph.tables[0];
+    // Verify the table compiles without database field since defaultProject is not set.
+    // The normalizeWarehouseTarget ensures that targets without database
+    // are correctly matched with warehouse state during the build phase, allowing incremental
+    // appends to work even when defaultProject is not specified in workflow_settings.yaml.
+    expect(compiledTable.type).equals("incremental");
+    expect(compiledTable.enumType).equals(dataform.TableType.INCREMENTAL);
+    expect(compiledTable.target.schema).equals("dataform");
+    expect(compiledTable.target.name).equals("incremental_table_without_default_project");
+    expect(compiledTable.target.database).equals("");
+  });
 });
