From bb5a9c3845fe2de0ed76e2c584314b634529c551 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Thu, 5 Mar 2026 14:51:18 +0100 Subject: [PATCH 01/39] [O2B-1545] Add GAQ summary models, adapters and migration --- lib/database/adapters/GaqSummaryAdapter.js | 56 +++++++++++++ .../adapters/GaqSummaryInvalidationAdapter.js | 46 +++++++++++ lib/database/adapters/index.js | 6 ++ ...0260223120000-create-gaq-summary-tables.js | 82 +++++++++++++++++++ lib/database/models/gaqSummary.js | 52 ++++++++++++ lib/database/models/gaqSummaryInvalidation.js | 37 +++++++++ lib/database/models/index.js | 4 + 7 files changed, 283 insertions(+) create mode 100644 lib/database/adapters/GaqSummaryAdapter.js create mode 100644 lib/database/adapters/GaqSummaryInvalidationAdapter.js create mode 100644 lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js create mode 100644 lib/database/models/gaqSummary.js create mode 100644 lib/database/models/gaqSummaryInvalidation.js diff --git a/lib/database/adapters/GaqSummaryAdapter.js b/lib/database/adapters/GaqSummaryAdapter.js new file mode 100644 index 0000000000..f069b0bfa7 --- /dev/null +++ b/lib/database/adapters/GaqSummaryAdapter.js @@ -0,0 +1,56 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +/** + * GaqSummaryAdapter + */ +class GaqSummaryAdapter { + /** + * Constructor + */ + constructor() { + this.toEntity = this.toEntity.bind(this); + } + + /** + * Converts the given database object to an entity object. + * + * @param {SequelizeGaqSummary} databaseObject Object to convert. + * @returns {GaqSummary} Converted entity object. + */ + toEntity(databaseObject) { + const { + dataPassId, + runNumber, + badEffectiveRunCoverage, + explicitlyNotBadEffectiveRunCoverage, + mcReproducible, + missingVerificationsCount, + undefinedQualityPeriodsCount, + computedAt, + } = databaseObject; + + return { + dataPassId, + runNumber, + badEffectiveRunCoverage, + explicitlyNotBadEffectiveRunCoverage, + mcReproducible, + missingVerificationsCount, + undefinedQualityPeriodsCount, + computedAt, + }; + } +} + +module.exports = { GaqSummaryAdapter }; diff --git a/lib/database/adapters/GaqSummaryInvalidationAdapter.js b/lib/database/adapters/GaqSummaryInvalidationAdapter.js new file mode 100644 index 0000000000..63d3ebce73 --- /dev/null +++ b/lib/database/adapters/GaqSummaryInvalidationAdapter.js @@ -0,0 +1,46 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +/** + * GaqSummaryInvalidationAdapter + */ +class GaqSummaryInvalidationAdapter { + /** + * Constructor + */ + constructor() { + this.toEntity = this.toEntity.bind(this); + } + + /** + * Converts the given database object to an entity object. + * + * @param {SequelizeGaqSummaryInvalidation} databaseObject Object to convert. + * @returns {GaqSummaryInvalidation} Converted entity object. + */ + toEntity(databaseObject) { + const { + dataPassId, + runNumber, + invalidatedAt, + } = databaseObject; + + return { + dataPassId, + runNumber, + invalidatedAt, + }; + } +} + +module.exports = { GaqSummaryInvalidationAdapter }; diff --git a/lib/database/adapters/index.js b/lib/database/adapters/index.js index 5ff6404b3d..97d22b927f 100644 --- a/lib/database/adapters/index.js +++ b/lib/database/adapters/index.js @@ -27,6 +27,8 @@ const EorReasonAdapter = require('./EorReasonAdapter'); const FlpRoleAdapter = require('./FlpRoleAdapter'); const { HostAdapter } = require('./HostAdapter.js'); const { GaqDetectorAdapter } = require('./GaqDetectorAdapter.js'); +const { GaqSummaryAdapter } = require('./GaqSummaryAdapter.js'); +const { GaqSummaryInvalidationAdapter } = require('./GaqSummaryInvalidationAdapter.js'); const { LhcFillAdapter } = require('./LhcFillAdapter.js'); const { LhcFillStatisticsAdapter } = require('./LhcFillStatisticsAdapter.js'); const LhcPeriodAdapter = require('./LhcPeriodAdapter'); @@ -63,6 +65,8 @@ const environmentHistoryItemAdapter = new EnvironmentHistoryItemAdapter(); const eorReasonAdapter = new EorReasonAdapter(); const flpRoleAdapter = new FlpRoleAdapter(); const gaqDetectorAdapter = new GaqDetectorAdapter(); +const gaqSummaryAdapter = new GaqSummaryAdapter(); +const gaqSummaryInvalidationAdapter = new GaqSummaryInvalidationAdapter(); const hostAdapter = new HostAdapter(); const lhcFillAdapter = new LhcFillAdapter(); const lhcFillStatisticsAdapter = new LhcFillStatisticsAdapter(); @@ -159,6 +163,8 @@ module.exports = { eorReasonAdapter, flpRoleAdapter, gaqDetectorAdapter, + gaqSummaryAdapter, + gaqSummaryInvalidationAdapter, hostAdapter, lhcFillAdapter, lhcFillStatisticsAdapter, diff --git a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js new file mode 100644 index 0000000000..9ead5d5f32 --- /dev/null +++ b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js @@ -0,0 +1,82 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + up: async (queryInterface, Sequelize) => queryInterface.sequelize.transaction(async (transaction) => { + await queryInterface.createTable('gaq_summaries', { + data_pass_id: { + type: Sequelize.INTEGER, + primaryKey: true, + allowNull: false, + references: { + model: 'data_passes', + key: 'id', + }, + }, + run_number: { + type: Sequelize.INTEGER, + primaryKey: true, + allowNull: false, + references: { + model: 'runs', + key: 'run_number', + }, + }, + bad_effective_run_coverage: { + type: Sequelize.FLOAT, + allowNull: false, + }, + explicitly_not_bad_effective_run_coverage: { + type: Sequelize.FLOAT, + allowNull: false, + }, + mc_reproducible: { + type: Sequelize.BOOLEAN, + allowNull: false, + }, + missing_verifications_count: { + type: Sequelize.INTEGER, + allowNull: false, + }, + undefined_quality_periods_count: { + type: Sequelize.INTEGER, + allowNull: false, + }, + computed_at: { + type: Sequelize.DATE(3), + allowNull: false, + }, + }, { transaction }); + + await queryInterface.createTable('gaq_summary_invalidations', { + data_pass_id: { + type: Sequelize.INTEGER, + primaryKey: true, + allowNull: false, + references: { + model: 'data_passes', + key: 'id', + }, + }, + run_number: { + type: Sequelize.INTEGER, + primaryKey: true, + allowNull: false, + references: { + model: 'runs', + key: 'run_number', + }, + }, + invalidated_at: { + type: Sequelize.DATE(3), + allowNull: false, + defaultValue: Sequelize.literal('CURRENT_TIMESTAMP(3)'), + }, + }, { transaction }); + }), + + down: async (queryInterface) => queryInterface.sequelize.transaction(async (transaction) => { + await queryInterface.dropTable('gaq_summary_invalidations', { transaction }); + await queryInterface.dropTable('gaq_summaries', { transaction }); + }), +}; diff --git a/lib/database/models/gaqSummary.js b/lib/database/models/gaqSummary.js new file mode 100644 index 0000000000..80e21f0145 --- /dev/null +++ b/lib/database/models/gaqSummary.js @@ -0,0 +1,52 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +module.exports = (sequelize) => { + const Sequelize = require('sequelize'); + + const GaqSummary = sequelize.define('GaqSummary', { + dataPassId: { + type: Sequelize.INTEGER, + }, + runNumber: { + type: Sequelize.INTEGER, + }, + badEffectiveRunCoverage: { + type: Sequelize.FLOAT, + }, + explicitlyNotBadEffectiveRunCoverage: { + type: Sequelize.FLOAT, + }, + mcReproducible: { + type: Sequelize.BOOLEAN, + }, + missingVerificationsCount: { + type: Sequelize.INTEGER, + }, + undefinedQualityPeriodsCount: { + type: Sequelize.INTEGER, + }, + computedAt: { + type: Sequelize.DATE(3), + }, + }, { tableName: 'gaq_summaries' }); + + GaqSummary.removeAttribute('id'); + + GaqSummary.associate = (models) => { + GaqSummary.belongsTo(models.Run, { foreignKey: 'runNumber', as: 'run' }); + GaqSummary.belongsTo(models.DataPass, { foreignKey: 'dataPassId', as: 'dataPass' }); + }; + + return GaqSummary; +}; diff --git a/lib/database/models/gaqSummaryInvalidation.js b/lib/database/models/gaqSummaryInvalidation.js new file mode 100644 index 0000000000..f99b5d0895 --- /dev/null +++ b/lib/database/models/gaqSummaryInvalidation.js @@ -0,0 +1,37 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +module.exports = (sequelize) => { + const Sequelize = require('sequelize'); + + const GaqSummaryInvalidation = sequelize.define('GaqSummaryInvalidation', { + dataPassId: { + type: Sequelize.INTEGER, + }, + runNumber: { + type: Sequelize.INTEGER, + }, + invalidatedAt: { + type: Sequelize.DATE(3), + }, + }, { tableName: 'gaq_summary_invalidations' }); + + GaqSummaryInvalidation.removeAttribute('id'); + + GaqSummaryInvalidation.associate = (models) => { + GaqSummaryInvalidation.belongsTo(models.Run, { foreignKey: 'runNumber', as: 'run' }); + GaqSummaryInvalidation.belongsTo(models.DataPass, { foreignKey: 'dataPassId', as: 'dataPass' }); + }; + + return GaqSummaryInvalidation; +}; diff --git a/lib/database/models/index.js b/lib/database/models/index.js index 87d793fac3..36736ce905 100644 --- a/lib/database/models/index.js +++ b/lib/database/models/index.js @@ -27,6 +27,8 @@ const EorReason = require('./eorreason'); const EpnRoleSession = require('./epnrolesession'); const FlpRole = require('./flprole'); const GaqDetector = require('./gaqDetector.js'); +const GaqSummary = require('./gaqSummary.js'); +const GaqSummaryInvalidation = require('./gaqSummaryInvalidation.js'); const Host = require('./host.js'); const LhcFill = require('./lhcFill'); const LhcFillStatistics = require('./lhcFillStatistics.js'); @@ -66,6 +68,8 @@ module.exports = (sequelize) => { EpnRoleSessionkey: EpnRoleSession(sequelize), FlpRole: FlpRole(sequelize), GaqDetector: GaqDetector(sequelize), + GaqSummary: GaqSummary(sequelize), + GaqSummaryInvalidation: GaqSummaryInvalidation(sequelize), Host: Host(sequelize), LhcFill: LhcFill(sequelize), LhcFillStatistics: LhcFillStatistics(sequelize), From 4fade0f12b6ac96baa67f298431e7025cafd7cca Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Mon, 16 Mar 2026 16:08:20 +0100 Subject: [PATCH 02/39] [O2B-1545] Add GAQ summary repositories and timestamps Add GaqSummaryRepository and GaqSummaryInvalidationRepository and export them from the repositories index. Update migration to replace the previous invalidated_at field with created_at and add updated_at to mirror default Sequelize tables. --- .../adapters/GaqSummaryInvalidationAdapter.js | 6 ++-- ...0260223120000-create-gaq-summary-tables.js | 7 +++- lib/database/models/gaqSummaryInvalidation.js | 3 -- .../GaqSummaryInvalidationRepository.js | 33 +++++++++++++++++++ .../repositories/GaqSummaryRepository.js | 33 +++++++++++++++++++ lib/database/repositories/index.js | 4 +++ 6 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 lib/database/repositories/GaqSummaryInvalidationRepository.js create mode 100644 lib/database/repositories/GaqSummaryRepository.js diff --git a/lib/database/adapters/GaqSummaryInvalidationAdapter.js b/lib/database/adapters/GaqSummaryInvalidationAdapter.js index 63d3ebce73..dbd76df3a8 100644 --- a/lib/database/adapters/GaqSummaryInvalidationAdapter.js +++ b/lib/database/adapters/GaqSummaryInvalidationAdapter.js @@ -32,13 +32,15 @@ class GaqSummaryInvalidationAdapter { const { dataPassId, runNumber, - invalidatedAt, + createdAt, + updatedAt, } = databaseObject; return { dataPassId, runNumber, - invalidatedAt, + createdAt, + updatedAt, }; } } diff --git a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js index 9ead5d5f32..7f08357a89 100644 --- a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js +++ b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js @@ -67,11 +67,16 @@ module.exports = { key: 'run_number', }, }, - invalidated_at: { + created_at: { type: Sequelize.DATE(3), allowNull: false, defaultValue: Sequelize.literal('CURRENT_TIMESTAMP(3)'), }, + updated_at: { + type: Sequelize.DATE(3), + allowNull: false, + defaultValue: Sequelize.literal('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'), + }, }, { transaction }); }), diff --git a/lib/database/models/gaqSummaryInvalidation.js b/lib/database/models/gaqSummaryInvalidation.js index f99b5d0895..3c0944b9bc 100644 --- a/lib/database/models/gaqSummaryInvalidation.js +++ b/lib/database/models/gaqSummaryInvalidation.js @@ -21,9 +21,6 @@ module.exports = (sequelize) => { runNumber: { type: Sequelize.INTEGER, }, - invalidatedAt: { - type: Sequelize.DATE(3), - }, }, { tableName: 'gaq_summary_invalidations' }); GaqSummaryInvalidation.removeAttribute('id'); diff --git a/lib/database/repositories/GaqSummaryInvalidationRepository.js b/lib/database/repositories/GaqSummaryInvalidationRepository.js new file mode 100644 index 0000000000..c6608f9e83 --- /dev/null +++ b/lib/database/repositories/GaqSummaryInvalidationRepository.js @@ -0,0 +1,33 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const { + models: { + GaqSummaryInvalidation, + }, +} = require('..'); +const Repository = require('./Repository'); + +/** + * GaqSummaryInvalidation repository + */ +class GaqSummaryInvalidationRepository extends Repository { + /** + * Creates a new `GaqSummaryInvalidationRepository` instance. + */ + constructor() { + super(GaqSummaryInvalidation); + } +} + +module.exports = new GaqSummaryInvalidationRepository(); diff --git a/lib/database/repositories/GaqSummaryRepository.js b/lib/database/repositories/GaqSummaryRepository.js new file mode 100644 index 0000000000..98986de513 --- /dev/null +++ b/lib/database/repositories/GaqSummaryRepository.js @@ -0,0 +1,33 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const { + models: { + GaqSummary, + }, +} = require('..'); +const Repository = require('./Repository'); + +/** + * GaqSummary repository + */ +class GaqSummaryRepository extends Repository { + /** + * Creates a new `GaqSummaryRepository` instance. + */ + constructor() { + super(GaqSummary); + } +} + +module.exports = new GaqSummaryRepository(); diff --git a/lib/database/repositories/index.js b/lib/database/repositories/index.js index 0c79279752..417ccd5375 100644 --- a/lib/database/repositories/index.js +++ b/lib/database/repositories/index.js @@ -27,6 +27,8 @@ const EnvironmentRepository = require('./EnvironmentRepository'); const EorReasonRepository = require('./EorReasonRepository'); const FlpRoleRepository = require('./FlpRoleRepository'); const GaqDetectorRepository = require('./GaqDetectorRepository.js'); +const GaqSummaryInvalidationRepository = require('./GaqSummaryInvalidationRepository.js'); +const GaqSummaryRepository = require('./GaqSummaryRepository.js'); const HostRepository = require('./HostRepository.js'); const LhcFillRepository = require('./LhcFillRepository'); const LhcFillStatisticsRepository = require('./LhcFillStatisticsRepository.js'); @@ -70,6 +72,8 @@ module.exports = { EorReasonRepository, FlpRoleRepository, GaqDetectorRepository, + GaqSummaryInvalidationRepository, + GaqSummaryRepository, HostRepository, LhcFillRepository, LhcFillStatisticsRepository, From 15afdaac85826129a0b127db26be1a3d355c5fdf Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 17 Mar 2026 10:15:14 +0100 Subject: [PATCH 03/39] [O2B-1545] Use default Sequelize createdAt/updatedAt instead of computedAt --- lib/database/adapters/GaqSummaryAdapter.js | 6 ++++-- .../v1/20260223120000-create-gaq-summary-tables.js | 8 +++++++- lib/database/models/gaqSummary.js | 3 --- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/database/adapters/GaqSummaryAdapter.js b/lib/database/adapters/GaqSummaryAdapter.js index f069b0bfa7..88d7394126 100644 --- a/lib/database/adapters/GaqSummaryAdapter.js +++ b/lib/database/adapters/GaqSummaryAdapter.js @@ -37,7 +37,8 @@ class GaqSummaryAdapter { mcReproducible, missingVerificationsCount, undefinedQualityPeriodsCount, - computedAt, + createdAt, + updatedAt, } = databaseObject; return { @@ -48,7 +49,8 @@ class GaqSummaryAdapter { mcReproducible, missingVerificationsCount, undefinedQualityPeriodsCount, - computedAt, + createdAt, + updatedAt, }; } } diff --git a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js index 7f08357a89..1e404d3645 100644 --- a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js +++ b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js @@ -42,9 +42,15 @@ module.exports = { type: Sequelize.INTEGER, allowNull: false, }, - computed_at: { + created_at: { type: Sequelize.DATE(3), allowNull: false, + defaultValue: Sequelize.literal('CURRENT_TIMESTAMP(3)'), + }, + updated_at: { + type: Sequelize.DATE(3), + allowNull: false, + defaultValue: Sequelize.literal('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'), }, }, { transaction }); diff --git a/lib/database/models/gaqSummary.js b/lib/database/models/gaqSummary.js index 80e21f0145..55b33b3521 100644 --- a/lib/database/models/gaqSummary.js +++ b/lib/database/models/gaqSummary.js @@ -36,9 +36,6 @@ module.exports = (sequelize) => { undefinedQualityPeriodsCount: { type: Sequelize.INTEGER, }, - computedAt: { - type: Sequelize.DATE(3), - }, }, { tableName: 'gaq_summaries' }); GaqSummary.removeAttribute('id'); From 363f09bed1b972877e491ca6f5c7a224617b8392 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 17 Mar 2026 17:03:35 +0100 Subject: [PATCH 04/39] [O2B-1545] Use mcReproducibleCoverage float instead of boolean Rename and change mcReproducible to be the coverage float not the boolean. --- lib/database/adapters/GaqSummaryAdapter.js | 4 ++-- .../migrations/v1/20260223120000-create-gaq-summary-tables.js | 4 ++-- lib/database/models/gaqSummary.js | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/database/adapters/GaqSummaryAdapter.js b/lib/database/adapters/GaqSummaryAdapter.js index 88d7394126..fd9ba4aa0e 100644 --- a/lib/database/adapters/GaqSummaryAdapter.js +++ b/lib/database/adapters/GaqSummaryAdapter.js @@ -34,7 +34,7 @@ class GaqSummaryAdapter { runNumber, badEffectiveRunCoverage, explicitlyNotBadEffectiveRunCoverage, - mcReproducible, + mcReproducibleCoverage, missingVerificationsCount, undefinedQualityPeriodsCount, createdAt, @@ -46,7 +46,7 @@ class GaqSummaryAdapter { runNumber, badEffectiveRunCoverage, explicitlyNotBadEffectiveRunCoverage, - mcReproducible, + mcReproducibleCoverage, missingVerificationsCount, undefinedQualityPeriodsCount, createdAt, diff --git a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js index 1e404d3645..8122607a37 100644 --- a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js +++ b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js @@ -30,8 +30,8 @@ module.exports = { type: Sequelize.FLOAT, allowNull: false, }, - mc_reproducible: { - type: Sequelize.BOOLEAN, + mc_reproducible_coverage: { + type: Sequelize.FLOAT, allowNull: false, }, missing_verifications_count: { diff --git a/lib/database/models/gaqSummary.js b/lib/database/models/gaqSummary.js index 55b33b3521..de91be13f2 100644 --- a/lib/database/models/gaqSummary.js +++ b/lib/database/models/gaqSummary.js @@ -30,6 +30,9 @@ module.exports = (sequelize) => { mcReproducible: { type: Sequelize.BOOLEAN, }, + mcReproducibleCoverage: { + type: Sequelize.FLOAT, + }, missingVerificationsCount: { type: Sequelize.INTEGER, }, From 002f3012a18f31ebaff85827935529156ce2909f Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 17 Mar 2026 17:49:41 +0100 Subject: [PATCH 05/39] [O2B-1545] Remove mcReproducible field Forgot to remove. --- lib/database/models/gaqSummary.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/database/models/gaqSummary.js b/lib/database/models/gaqSummary.js index de91be13f2..f3338e5557 100644 --- a/lib/database/models/gaqSummary.js +++ b/lib/database/models/gaqSummary.js @@ -27,9 +27,6 @@ module.exports = (sequelize) => { explicitlyNotBadEffectiveRunCoverage: { type: Sequelize.FLOAT, }, - mcReproducible: { - type: Sequelize.BOOLEAN, - }, mcReproducibleCoverage: { type: Sequelize.FLOAT, }, From 5bebbb5ac73f011462d327823b0e6c5f1ace192e Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 17 Mar 2026 09:24:00 +0100 Subject: [PATCH 06/39] [O2B-1563] Invalidate GAQ summaries on related changes Add GAQ summary invalidation whenever underlying data affecting GAQ changes. These changes ensure GAQ summary caches/records are marked for recomputation whenever detectors, QC flags, or run QC times that influence GAQ summaries are modified. --- .../services/gaq/GaqDetectorsService.js | 16 +++++- .../qualityControlFlag/QcFlagService.js | 50 +++++++++++++++++-- lib/server/services/run/updateRun.js | 25 +++++++++- 3 files changed, 85 insertions(+), 6 deletions(-) diff --git a/lib/server/services/gaq/GaqDetectorsService.js b/lib/server/services/gaq/GaqDetectorsService.js index 1ac2a0dc7e..6c19d1cd4a 100644 --- a/lib/server/services/gaq/GaqDetectorsService.js +++ b/lib/server/services/gaq/GaqDetectorsService.js @@ -12,7 +12,7 @@ */ const { gaqDetectorAdapter, detectorAdapter } = require('../../../database/adapters'); -const { GaqDetectorRepository, RunRepository, DetectorRepository } = require('../../../database/repositories'); +const { GaqDetectorRepository, RunRepository, DetectorRepository, GaqSummaryInvalidationRepository } = require('../../../database/repositories'); const { BadParameterError } = require('../../errors/BadParameterError'); const { dataSource } = require('../../../database/DataSource.js'); const { Op } = require('sequelize'); @@ -57,6 +57,13 @@ class GaqDetectorService { .flatMap((runNumber) => detectorIds .map((detectorId) => ({ dataPassId, runNumber, detectorId }))); const createdEntries = await GaqDetectorRepository.insertAll(gaqEntries); + + // Invalidate GAQ summaries for all affected runs + await Promise.all(runNumbers.map((runNumber) => GaqSummaryInvalidationRepository.upsert({ + dataPassId, + runNumber, + }))); + return createdEntries.map(gaqDetectorAdapter.toEntity); }); } @@ -101,6 +108,13 @@ class GaqDetectorService { .flatMap(({ runNumber, detectors }) => detectors .map(({ id: detectorId }) => ({ dataPassId, runNumber, detectorId }))); const createdEntries = await GaqDetectorRepository.insertAll(gaqEntries); + + // Invalidate GAQ summaries for all affected runs + await Promise.all(runNumbers.map((runNumber) => GaqSummaryInvalidationRepository.upsert({ + dataPassId, + runNumber, + }))); + return createdEntries.map(gaqDetectorAdapter.toEntity); }, { transaction }); } diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 3d34e1446b..3533716572 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -20,6 +20,7 @@ const { RunRepository, QcFlagVerificationRepository, QcFlagEffectivePeriodRepository, + GaqSummaryInvalidationRepository, }, } = require('../../../database/index.js'); const { dataSource } = require('../../../database/DataSource.js'); @@ -209,6 +210,14 @@ class QcFlagService { ], }); + if (dataPass) { + // Invalidate GAQ summary for the dataPass and runNumber of the created flag + await GaqSummaryInvalidationRepository.upsert({ + dataPassId: dataPassIdentifier, + runNumber, + }); + } + createdFlags.push(qcFlagAdapter.toEntity(createdFlag)); } catch (error) { this._logger.warnMessage(`Failed to create QC flag with properties: ${JSON.stringify(qcFlag)}. Error: ${error}`); @@ -284,6 +293,15 @@ class QcFlagService { { const { id, from, to, origin, createdById, runNumber, dplDetectorId, flagTypeId, createdAt } = qcFlag; + + if (dataPassId) { + // Invalidate GAQ summary for the dataPass and runNumber of the created flag + await GaqSummaryInvalidationRepository.upsert({ + dataPassId: dataPassId, + runNumber, + }); + } + const qcFlagPropertiesToLog = { id, from, @@ -338,8 +356,8 @@ class QcFlagService { await QcFlagEffectivePeriodRepository.removeAll({ where: { id: { [Op.in]: effectivePeriodIds } } }); // Sequelize update requires a where and can't work only using association - const qcFlagIds = (await QcFlagRepository.findAll({ - attributes: ['id'], + const qcFlagIdsToRunNumbers = (await QcFlagRepository.findAll({ + attributes: ['id', 'runNumber'], include: { association: 'dataPasses', attributes: [], @@ -349,13 +367,23 @@ class QcFlagService { }, }, raw: true, - })).map(({ id }) => id); + })).map(({ id, runNumber }) => ({ id, runNumber })); + + const qcFlagIds = qcFlagIdsToRunNumbers.map(({ id }) => id); + const runNumbers = new Set(); + qcFlagIdsToRunNumbers.map(({ runNumber }) => runNumbers.add(runNumber)); await QcFlagRepository.updateAll( { deleted: true }, { where: { id: qcFlagIds } }, ); + // Invalidate GAQ summary for the dataPass and all runNumbers of the deleted flags + await Promise.all(Array.from(runNumbers).map((runNumber) => GaqSummaryInvalidationRepository.upsert({ + dataPassId, + runNumber, + }))); + return qcFlagIds.length; }); } @@ -389,7 +417,21 @@ class QcFlagService { createdById: user.id, }); - return await this.getOneOrFail(flagId); + const updatedQcFlag = await this.getOneOrFail(flagId); + + // Get data pass id for the flag (if exists) to invalidate only related GAQ summaries + const dataPassQcFlag = await DataPassQcFlagRepository.findOne({ where: { qualityControlFlagId: flagId } }); + const dataPassId = dataPassQcFlag?.dataPassId; + + // Invalidate GAQ summary if it's the first verification + if (dataPassId && (!qcFlag.verifications || qcFlag.verifications.length === 0)) { + await GaqSummaryInvalidationRepository.upsert({ + dataPassId, + runNumber: updatedQcFlag.runNumber, + }); + } + + return updatedQcFlag; }, { transaction }); } diff --git a/lib/server/services/run/updateRun.js b/lib/server/services/run/updateRun.js index 2e8b50048c..98c7247c53 100644 --- a/lib/server/services/run/updateRun.js +++ b/lib/server/services/run/updateRun.js @@ -12,6 +12,8 @@ */ const RunRepository = require('../../../database/repositories/RunRepository.js'); +const GaqSummaryInvalidationRepository = require('../../../database/repositories/GaqSummaryInvalidationRepository.js'); +const DataPassRunRepository = require('../../../database/repositories/DataPassRunRepository.js'); const { getRunOrFail } = require('./getRunOrFail.js'); const { utilities: { TransactionHelper } } = require('../../../database'); const { checkLhcFill } = require('../../../usecases/lhcFill/checkLhcFill.js'); @@ -82,7 +84,12 @@ exports.updateRun = async (identifier, payload, transaction) => { } // Store the run quality to create a log if it changed - const previousRun = { runQuality: runModel?.runQuality, calibrationStatus: runModel?.calibrationStatus }; + const previousRun = { + runQuality: runModel?.runQuality, + calibrationStatus: runModel?.calibrationStatus, + qcTimeStart: runModel?.qcTimeStart, + qcTimeEnd: runModel?.qcTimeEnd, + }; runPatch.definition = runPatch.definition ?? getRunDefinition({ ...runModel.dataValues, @@ -204,6 +211,22 @@ exports.updateRun = async (identifier, payload, transaction) => { } } + // Need to reload the runModel as qcTimeStart and qcTimeEnd are virtual columns not in the patch so will stay stale after update + await runModel.reload(); + if (previousRun.qcTimeStart?.getTime() !== runModel.qcTimeStart?.getTime() + || previousRun.qcTimeEnd?.getTime() !== runModel.qcTimeEnd?.getTime()) { + const dataPassRuns = await DataPassRunRepository.findAll({ + attributes: ['dataPassId'], + where: { runNumber: runModel.runNumber }, + }); + for (const { dataPassId } of dataPassRuns) { + await GaqSummaryInvalidationRepository.upsert({ + dataPassId, + runNumber: runModel.runNumber, + }); + } + } + return runModel; }, { transaction }); }; From ee55f5972849abe0d6b35abbba346bbbcd32ac36 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 17 Mar 2026 16:06:51 +0100 Subject: [PATCH 07/39] Add GAQ worker and summary recalculation Introduce background processing for GAQ summary invalidations and wire it into the app scheduler. Changes include: - Add gaq config to services config. - Move GaqService to lib/server/services/gaq and update imports across controllers/use-cases/tests. - Extend GaqService to pop invalidations and recalculate summaries within a transaction. - Add GaqWorker that guards concurrent runs and calls GaqService to process a batch of invalid summaries. - Schedule the GaqWorker in application startup when GAQ recalculation is enabled. - Add a soft-delete filter (where: { deleted: false }) when querying QC flags to map ids to run numbers as otherwise summary invalidations occur even on already deleted flags. These changes enable periodic recalculation of GAQ summaries when invalidations are queued in the table. --- lib/application.js | 14 +++++- lib/config/services.js | 6 +++ lib/server/controllers/qcFlag.controller.js | 2 +- .../{qualityControlFlag => gaq}/GaqService.js | 45 ++++++++++++++++++- lib/server/services/gaq/GaqWorker.js | 35 +++++++++++++++ .../qualityControlFlag/QcFlagService.js | 1 + lib/usecases/run/GetAllRunsUseCase.js | 2 +- .../qualityControlFlag/QcFlagService.test.js | 2 +- 8 files changed, 102 insertions(+), 5 deletions(-) rename lib/server/services/{qualityControlFlag => gaq}/GaqService.js (73%) create mode 100644 lib/server/services/gaq/GaqWorker.js diff --git a/lib/application.js b/lib/application.js index 70a4ac6839..0f24120f48 100644 --- a/lib/application.js +++ b/lib/application.js @@ -15,7 +15,7 @@ const database = require('./database'); const { webUiServer } = require('./server'); const { GRPCConfig, ServicesConfig } = require('./config'); -const { userCertificate, monalisa: monalisaConfig, ccdb: ccdbConfig, enableHousekeeping } = ServicesConfig; +const { userCertificate, monalisa: monalisaConfig, ccdb: ccdbConfig, gaq: gaqConfig, enableHousekeeping } = ServicesConfig; const { handleLostRunsAndEnvironments } = require('./server/services/housekeeping/handleLostRunsAndEnvironments.js'); const { isInTestMode } = require('./utilities/env-utils.js'); const { ScheduledProcessesManager } = require('./server/services/ScheduledProcessesManager.js'); @@ -27,6 +27,7 @@ const { AliEcsSynchronizer } = require('./server/kafka/AliEcsSynchronizer.js'); const { environmentService } = require('./server/services/environment/EnvironmentService.js'); const { runService } = require('./server/services/run/RunService.js'); const { CcdbSynchronizer } = require('./server/externalServicesSynchronization/ccdb/CcdbSynchronizer.js'); +const { GaqWorker } = require('./server/services/gaq/GaqWorker.js'); const { promises: fs } = require('fs'); const { MonAlisaClient } = require('./server/externalServicesSynchronization/monalisa/MonAlisaClient.js'); const https = require('https'); @@ -131,6 +132,17 @@ class BookkeepingApplication { }, ); } + + if (gaqConfig.enableRecalculation) { + const gaqWorker = new GaqWorker(); + this.scheduledProcessesManager.schedule( + () => gaqWorker.recalculateGaqSummaries(gaqConfig.batchSize), + { + wait: 10 * 1000, + every: gaqConfig.recalculationPeriod, + }, + ); + } } catch (error) { this._logger.errorMessage(`Error while starting: ${error}`); return this.stop(); diff --git a/lib/config/services.js b/lib/config/services.js index 9e535881a9..2c9f941c6a 100644 --- a/lib/config/services.js +++ b/lib/config/services.js @@ -68,4 +68,10 @@ exports.services = { synchronizationPeriod: Number(CCDB_SYNCHRONIZATION_PERIOD) || 24 * 60 * 60 * 1000, // 1d in milliseconds runInfoUrl: CCDB_RUN_INFO_URL, }, + + gaq: { + enableRecalculation: process.env?.GAQ_ENABLE_RECALCULATION?.toLowerCase() === 'true' || true, + recalculationPeriod: Number(process.env?.GAQ_RECALCULATION_PERIOD) || 60 * 1000, // 1m in milliseconds + batchSize: Number(process.env?.GAQ_RECALCULATION_BATCH_SIZE) || 1, + }, }; diff --git a/lib/server/controllers/qcFlag.controller.js b/lib/server/controllers/qcFlag.controller.js index 7a088a3eb3..b7e4cf15f3 100644 --- a/lib/server/controllers/qcFlag.controller.js +++ b/lib/server/controllers/qcFlag.controller.js @@ -21,7 +21,7 @@ const { PaginationDto } = require('../../domain/dtos'); const { ApiConfig } = require('../../config'); const { countedItemsToHttpView } = require('../utilities/countedItemsToHttpView'); const { qcFlagService } = require('../services/qualityControlFlag/QcFlagService.js'); -const { gaqService } = require('../services/qualityControlFlag/GaqService.js'); +const { gaqService } = require('../services/gaq/GaqService.js'); const { qcFlagSummaryService } = require('../services/qualityControlFlag/QcFlagSummaryService.js'); const qcFlagFilterDTO = Joi.object({ diff --git a/lib/server/services/qualityControlFlag/GaqService.js b/lib/server/services/gaq/GaqService.js similarity index 73% rename from lib/server/services/qualityControlFlag/GaqService.js rename to lib/server/services/gaq/GaqService.js index 422e48bcac..5336ac68b4 100644 --- a/lib/server/services/qualityControlFlag/GaqService.js +++ b/lib/server/services/gaq/GaqService.js @@ -28,15 +28,24 @@ */ const { getOneDataPassOrFail } = require('../dataPasses/getOneDataPassOrFail.js'); -const { QcFlagRepository } = require('../../../database/repositories/index.js'); +const { QcFlagRepository, GaqSummaryRepository, GaqSummaryInvalidationRepository } = require('../../../database/repositories/index.js'); const { qcFlagAdapter } = require('../../../database/adapters/index.js'); const { Op } = require('sequelize'); const { QcSummarProperties } = require('../../../domain/enums/QcSummaryProperties.js'); +const { dataSource } = require('../../../database/DataSource.js'); +const { LogManager } = require('@aliceo2/web-ui'); /** * Globally aggregated quality (QC flags aggregated for a predefined list of detectors per runs) service */ class GaqService { + /** + * Constructor + */ + constructor() { + this._logger = LogManager.getLogger('GAQ_SERVICE'); + } + /** * Get GAQ summary * @@ -113,6 +122,40 @@ class GaqService { contributingFlags: contributingFlagIds.map((id) => idToFlag[id]), })); } + + /** + * Calculate and store GAQ summary for given data pass and run + * @param {number} dataPassId id of data pass + * @param {number} runNumber run number + * @return {Promise} promise + */ + async calculateAndStoreGaqSummary(dataPassId, runNumber) { + const summary = await this.getSummary(dataPassId, { runNumber }); + await GaqSummaryRepository.upsert({ dataPassId, runNumber, ...summary }); + } + + /** + * Remove invalid GAQ summaries and recalculate them + * @param {number} batchSize maximum number of invalid summaries to process + * @return {Promise} promise + */ + async popNInvalidSummaryAndRecalculate(batchSize = 1) { + const invalidCount = await GaqSummaryInvalidationRepository.count(); + const remaining = Math.min(batchSize, invalidCount); + if (remaining === 0) { + return; + } + await dataSource.transaction(async () => { + for (let i = 0; i < remaining; i++) { + const invalidation = await GaqSummaryInvalidationRepository.removeOne({ where: {}, order: [['createdAt', 'ASC']] }); + if (!invalidation) { + break; + } + const { dataPassId, runNumber } = invalidation; + await this.calculateAndStoreGaqSummary(dataPassId, runNumber); + } + }); + } } exports.GaqService = GaqService; diff --git a/lib/server/services/gaq/GaqWorker.js b/lib/server/services/gaq/GaqWorker.js new file mode 100644 index 0000000000..1fa01920c0 --- /dev/null +++ b/lib/server/services/gaq/GaqWorker.js @@ -0,0 +1,35 @@ +const { gaqService } = require('../../services/gaq/GaqService.js'); +const { LogManager } = require('@aliceo2/web-ui'); + +/** + * Worker responsible for processing pending GAQ summary invalidations + */ +class GaqWorker { + /** + * Constructor + */ + constructor() { + this._logger = LogManager.getLogger(GaqWorker.name); + } + + /** + * Process pending GAQ summary invalidations. Skips if a previous call is still in progress. + * @param {number} batchSize number of invalid summaries to recalculate + * @return {Promise} promise + */ + async recalculateGaqSummaries(batchSize) { + if (this._isSynchronizing) { + return; + } + this._isSynchronizing = true; + try { + await gaqService.popNInvalidSummaryAndRecalculate(batchSize); + } catch (error) { + this._logger.errorMessage(`Error recalculating GAQ summaries: ${error.message}\n${error.stack}`); + } finally { + this._isSynchronizing = false; + } + } +} + +exports.GaqWorker = GaqWorker; diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 3533716572..6ae491966c 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -358,6 +358,7 @@ class QcFlagService { // Sequelize update requires a where and can't work only using association const qcFlagIdsToRunNumbers = (await QcFlagRepository.findAll({ attributes: ['id', 'runNumber'], + where: { deleted: false }, include: { association: 'dataPasses', attributes: [], diff --git a/lib/usecases/run/GetAllRunsUseCase.js b/lib/usecases/run/GetAllRunsUseCase.js index df1b5f7f5b..3fd43b8f76 100644 --- a/lib/usecases/run/GetAllRunsUseCase.js +++ b/lib/usecases/run/GetAllRunsUseCase.js @@ -20,7 +20,7 @@ const sequelize = require('sequelize'); const { EorReasonRepository } = require('../../database/repositories'); const { PhysicalConstant } = require('../../domain/enums/PhysicalConstant'); const { BadParameterError } = require('../../server/errors/BadParameterError'); -const { gaqService } = require('../../server/services/qualityControlFlag/GaqService.js'); +const { gaqService } = require('../../server/services/gaq/GaqService.js'); const { qcFlagSummaryService } = require('../../server/services/qualityControlFlag/QcFlagSummaryService.js'); const { DetectorType } = require('../../domain/enums/DetectorTypes.js'); const { unpackNumberRange } = require('../../utilities/rangeUtils.js'); diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index f4c533444f..e62fd06b4e 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -21,7 +21,7 @@ const { Op } = require('sequelize'); const { qcFlagAdapter } = require('../../../../../lib/database/adapters'); const { runService } = require('../../../../../lib/server/services/run/RunService'); const { gaqDetectorService } = require('../../../../../lib/server/services/gaq/GaqDetectorsService'); -const { gaqService } = require('../../../../../lib/server/services/qualityControlFlag/GaqService.js'); +const { gaqService } = require('../../../../../lib/server/services/gaq/GaqService.js'); const { qcFlagSummaryService } = require('../../../../../lib/server/services/qualityControlFlag/QcFlagSummaryService.js'); const { dataPassService } = require('../../../../../lib/server/services/dataPasses/DataPassService.js'); From 530216b98a8e8484a6cd2e5a1e77c049a5408049 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 17 Mar 2026 17:26:13 +0100 Subject: [PATCH 08/39] [O2B-1564] Create computeSummary to separate getting and computing --- lib/server/services/gaq/GaqService.js | 36 ++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/lib/server/services/gaq/GaqService.js b/lib/server/services/gaq/GaqService.js index 5336ac68b4..178a810dee 100644 --- a/lib/server/services/gaq/GaqService.js +++ b/lib/server/services/gaq/GaqService.js @@ -92,6 +92,37 @@ class GaqService { return Object.fromEntries(gaqSummary); } + /** + * Find GAQ summary for given data pass and run. Returns null if no summary can be computed (e.g. no QC flags in GAQ periods) + * @param {number} dataPassId id of data pass + * @param {number} runNumber run number + * @return {Promise} promise of GAQ summary or null if it can't be computed + */ + async _computeSummary(dataPassId, runNumber) { + const gaqCoverages = await QcFlagRepository.getGaqCoverages(dataPassId, runNumber); + const entry = gaqCoverages[runNumber]; + if (!entry) { + return null; + } + + const { + badCoverage, + mcReproducibleCoverage, + goodCoverage, + flagsIds, + verifiedFlagsIds, + undefinedQualityPeriodsCount, + } = entry; + + return { + badEffectiveRunCoverage: badCoverage + mcReproducibleCoverage, + explicitlyNotBadEffectiveRunCoverage: goodCoverage, + mcReproducibleCoverage, + missingVerificationsCount: flagsIds.length - verifiedFlagsIds.length, + undefinedQualityPeriodsCount, + }; + } + /** * Find QC flags in GAQ effective periods for given data pass and run * @@ -130,7 +161,10 @@ class GaqService { * @return {Promise} promise */ async calculateAndStoreGaqSummary(dataPassId, runNumber) { - const summary = await this.getSummary(dataPassId, { runNumber }); + const summary = await this._computeSummary(dataPassId, runNumber); + if (!summary) { + return; + } await GaqSummaryRepository.upsert({ dataPassId, runNumber, ...summary }); } From e1f3223e6b8c8f5f6414a999962e56cbf72ba78c Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 17 Mar 2026 17:41:46 +0100 Subject: [PATCH 09/39] [O2B-1564] Exclude reproducible coverage from badEffectiveRunCoverage --- lib/server/services/gaq/GaqService.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/server/services/gaq/GaqService.js b/lib/server/services/gaq/GaqService.js index 178a810dee..82edf93ed5 100644 --- a/lib/server/services/gaq/GaqService.js +++ b/lib/server/services/gaq/GaqService.js @@ -115,7 +115,7 @@ class GaqService { } = entry; return { - badEffectiveRunCoverage: badCoverage + mcReproducibleCoverage, + badEffectiveRunCoverage: badCoverage, explicitlyNotBadEffectiveRunCoverage: goodCoverage, mcReproducibleCoverage, missingVerificationsCount: flagsIds.length - verifiedFlagsIds.length, From 16d3563ef0813e8c10de405ef7d9cda50eed8b3c Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Wed, 18 Mar 2026 16:34:34 +0100 Subject: [PATCH 10/39] [O2B-1545] Rename GAQ summary coverage fields --- lib/database/adapters/GaqSummaryAdapter.js | 8 ++++---- .../v1/20260223120000-create-gaq-summary-tables.js | 4 ++-- lib/database/models/gaqSummary.js | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/database/adapters/GaqSummaryAdapter.js b/lib/database/adapters/GaqSummaryAdapter.js index fd9ba4aa0e..77ff939d69 100644 --- a/lib/database/adapters/GaqSummaryAdapter.js +++ b/lib/database/adapters/GaqSummaryAdapter.js @@ -32,8 +32,8 @@ class GaqSummaryAdapter { const { dataPassId, runNumber, - badEffectiveRunCoverage, - explicitlyNotBadEffectiveRunCoverage, + badRunCoverage, + explicitlyNotBadRunCoverage, mcReproducibleCoverage, missingVerificationsCount, undefinedQualityPeriodsCount, @@ -44,8 +44,8 @@ class GaqSummaryAdapter { return { dataPassId, runNumber, - badEffectiveRunCoverage, - explicitlyNotBadEffectiveRunCoverage, + badRunCoverage, + explicitlyNotBadRunCoverage, mcReproducibleCoverage, missingVerificationsCount, undefinedQualityPeriodsCount, diff --git a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js index 8122607a37..b734f32e32 100644 --- a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js +++ b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js @@ -22,11 +22,11 @@ module.exports = { key: 'run_number', }, }, - bad_effective_run_coverage: { + bad_run_coverage: { type: Sequelize.FLOAT, allowNull: false, }, - explicitly_not_bad_effective_run_coverage: { + explicitly_not_bad_run_coverage: { type: Sequelize.FLOAT, allowNull: false, }, diff --git a/lib/database/models/gaqSummary.js b/lib/database/models/gaqSummary.js index f3338e5557..480a9a4943 100644 --- a/lib/database/models/gaqSummary.js +++ b/lib/database/models/gaqSummary.js @@ -21,10 +21,10 @@ module.exports = (sequelize) => { runNumber: { type: Sequelize.INTEGER, }, - badEffectiveRunCoverage: { + badRunCoverage: { type: Sequelize.FLOAT, }, - explicitlyNotBadEffectiveRunCoverage: { + explicitlyNotBadRunCoverage: { type: Sequelize.FLOAT, }, mcReproducibleCoverage: { From 87afd8c256ec966b5bde13dc40dc03c6f379b9d3 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Wed, 18 Mar 2026 16:51:55 +0100 Subject: [PATCH 11/39] [O2B-1564] Rename coverage fields to match table cols in GaqService output --- lib/server/services/gaq/GaqService.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/server/services/gaq/GaqService.js b/lib/server/services/gaq/GaqService.js index 82edf93ed5..79997ad06c 100644 --- a/lib/server/services/gaq/GaqService.js +++ b/lib/server/services/gaq/GaqService.js @@ -115,8 +115,8 @@ class GaqService { } = entry; return { - badEffectiveRunCoverage: badCoverage, - explicitlyNotBadEffectiveRunCoverage: goodCoverage, + badRunCoverage: badCoverage, + explicitlyNotBadRunCoverage: goodCoverage, mcReproducibleCoverage, missingVerificationsCount: flagsIds.length - verifiedFlagsIds.length, undefinedQualityPeriodsCount, From 170c1b37f6556e0e39c8f9b54e62bf737523587b Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 31 Mar 2026 14:41:48 +0200 Subject: [PATCH 12/39] [O2B-1563] Add GAQ summary invalidation trigger tests Validate GAQ summary invalidation behaviour on QC flag create/verify/delete, deleteAllForDataPass, explicit/default GAQ detector changes, and run QC time updates. --- .../server/services/gaq/GaqSummary.test.js | 121 ++++++++++++++++++ test/lib/server/services/gaq/index.js | 2 + test/lib/server/services/index.js | 4 +- 3 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 test/lib/server/services/gaq/GaqSummary.test.js diff --git a/test/lib/server/services/gaq/GaqSummary.test.js b/test/lib/server/services/gaq/GaqSummary.test.js new file mode 100644 index 0000000000..bffd5c5f05 --- /dev/null +++ b/test/lib/server/services/gaq/GaqSummary.test.js @@ -0,0 +1,121 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const { expect } = require('chai'); +const { resetDatabaseContent } = require('../../../../utilities/resetDatabaseContent.js'); +const { repositories: { GaqSummaryInvalidationRepository } } = require('../../../../../lib/database'); +const { qcFlagService } = require('../../../../../lib/server/services/qualityControlFlag/QcFlagService.js'); +const { gaqDetectorService } = require('../../../../../lib/server/services/gaq/GaqDetectorsService.js'); +const { updateRun } = require('../../../../../lib/server/services/run/updateRun.js'); + +module.exports = () => { + // Test resets the database before running and clears the invalidation table between each case + before(async () => { + await resetDatabaseContent(); + }); + + const relations = { user: { roles: ['admin'], externalUserId: 1 } }; + const dataPassId = 4; // LHC22a_apass2 + const runNumber = 56; + + /** + * Check whether an invalidation entry exists for a given data pass and run + * + * @param {number} expectedDataPassId + * @param {number} expectedRunNumber + * @param {boolean} toBeNull + * + * @return {Promise} + */ + const expectInvalidation = async (expectedDataPassId, expectedRunNumber, toBeNull = false) => { + const invalidation = await GaqSummaryInvalidationRepository.findOne({ + where: { dataPassId: expectedDataPassId, runNumber: expectedRunNumber }, + }); + if (toBeNull) { + expect(invalidation, `Expected no invalidation for dataPassId=${expectedDataPassId} runNumber=${expectedRunNumber}`).to.be.null; + } else { + expect(invalidation, `Expected invalidation for dataPassId=${expectedDataPassId} runNumber=${expectedRunNumber}`).to.not.be.null; + } + }; + + describe("GAQ Summary Invalidation", async () => { + afterEach(async () => { + await GaqSummaryInvalidationRepository.removeAll({ truncate: true }); + }); + + it('should invalidate GAQ summary when a QC flag is created for a data pass', async () => { + await qcFlagService.create( + [{ from: null, to: null, flagTypeId: 3 }], + { runNumber, detectorIdentifier: { detectorId: 7 }, dataPassIdentifier: { id: dataPassId } }, + relations, + ); + + await expectInvalidation(dataPassId, runNumber); + }); + + it('should invalidate GAQ summary when a QC flag is verified for the first time for a data pass', async () => { + const flagId = 8; // Seeded flag in data pass 4, run 100, with no verifications + + await qcFlagService.verifyFlag({ flagId }, relations); + + await expectInvalidation(dataPassId, 100); + + // Clear invalidation + await GaqSummaryInvalidationRepository.removeAll({ where: { dataPassId, runNumber: 100 } }); + + // Verify again to check that no new invalidation is created when the flag is already verified + await qcFlagService.verifyFlag({ flagId }, relations); + await expectInvalidation(dataPassId, 100, true); + }); + + it('should invalidate GAQ summary when a QC flag is deleted for a data pass', async () => { + const flagId = 9; // Seeded flag in data pass 4, run 105, with no verifications (so deletion is allowed) + await qcFlagService.delete(flagId); + + await expectInvalidation(dataPassId, 105); + }); + + it('should invalidate GAQ summary for all runs when all QC flags are deleted for a data pass', async () => { + await qcFlagService.deleteAllForDataPass(dataPassId); + + await expectInvalidation(dataPassId, 100); + await expectInvalidation(dataPassId, 105); + }); + + it('should invalidate GAQ summary when GAQ detectors are explicitly set for a data pass and run', async () => { + const gaqDataPassId = 3; // LHC22a_apass1 (has run 56 and detectors set up in GaqDetectorService tests) + const detectorIds = [4, 7]; + + await gaqDetectorService.setGaqDetectors(gaqDataPassId, [runNumber], detectorIds); + + await expectInvalidation(gaqDataPassId, runNumber); + }); + + it('should invalidate GAQ summary when default GAQ detectors are used for a data pass and run', async () => { + const gaqDataPassId = 3; + + await gaqDetectorService.useDefaultGaqDetectors(gaqDataPassId, [runNumber]); + + await expectInvalidation(gaqDataPassId, runNumber); + }); + + it('should invalidate GAQ summary when the QC time of a run changes', async () => { + await updateRun( + { runNumber }, + { runPatch: { timeTrgStart: new Date('2019-08-08 20:30:00') } }, + ); + + await expectInvalidation(dataPassId, runNumber); + }); + }); +}; diff --git a/test/lib/server/services/gaq/index.js b/test/lib/server/services/gaq/index.js index 985f80fe84..93e10b2ac6 100644 --- a/test/lib/server/services/gaq/index.js +++ b/test/lib/server/services/gaq/index.js @@ -12,7 +12,9 @@ */ const GaqDetectorServiceSuite = require('./GaqDetectorService.test.js'); +const GaqSummarySuite = require('./GaqSummary.test.js'); module.exports = () => { describe('GaqDetectorService', GaqDetectorServiceSuite); + describe('GaqSummary', GaqSummarySuite); }; diff --git a/test/lib/server/services/index.js b/test/lib/server/services/index.js index 01df33fcaf..38aabbf6dd 100644 --- a/test/lib/server/services/index.js +++ b/test/lib/server/services/index.js @@ -32,7 +32,7 @@ const UserSuite = require('./user/index.js'); const SimulationPassesSuite = require('./simulationPasses/index.js'); const QcFlagsSuite = require('./qualityControlFlag/index.js'); const CtpTriggerCountersSuite = require('./ctpTriggerCounters/index.js'); -const GaqDetectorSuite = require('./gaq'); +const GaqSuite = require('./gaq'); module.exports = () => { before(resetDatabaseContent); @@ -46,7 +46,7 @@ module.exports = () => { describe('Environment history item', EnvironmentHistoryItemSuite); describe('EOS report', EosReportSuite); describe('Flp role', FlpRoleSuite); - describe('GaqDetector', GaqDetectorSuite); + describe('GAQ', GaqSuite); describe('LHC fill suite', LhcFillSuite); describe('Logs', LogSuite); describe('RunType', RunTypeSuite); From 78a2181986fd52f57a28435efdf160897919c9c1 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 31 Mar 2026 14:46:57 +0200 Subject: [PATCH 13/39] [O2B-1563] Use dataPass.id in GAQ invalidation TYPO --- lib/server/services/qualityControlFlag/QcFlagService.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 3533716572..3f0ca96d79 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -213,7 +213,7 @@ class QcFlagService { if (dataPass) { // Invalidate GAQ summary for the dataPass and runNumber of the created flag await GaqSummaryInvalidationRepository.upsert({ - dataPassId: dataPassIdentifier, + dataPassId: dataPass.id, runNumber, }); } From abb263b8c3afd4d3e064960f5b504f654bd6567f Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Wed, 1 Apr 2026 17:11:00 +0200 Subject: [PATCH 14/39] Make GAQ worker singleton and add pause/resume Converted gaqWorker to a singleton. This allows adding of pause() and resume() methods to prevent the worker from processing invalidated summaries during test execution. Reduced the default GAQ recalculation period from 1 minute to 10 seconds to improve test suite performance. --- lib/application.js | 3 +-- lib/config/services.js | 2 +- lib/server/services/gaq/GaqWorker.js | 26 +++++++++++++++++++++++--- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/lib/application.js b/lib/application.js index 0f24120f48..f7d64dce01 100644 --- a/lib/application.js +++ b/lib/application.js @@ -27,7 +27,7 @@ const { AliEcsSynchronizer } = require('./server/kafka/AliEcsSynchronizer.js'); const { environmentService } = require('./server/services/environment/EnvironmentService.js'); const { runService } = require('./server/services/run/RunService.js'); const { CcdbSynchronizer } = require('./server/externalServicesSynchronization/ccdb/CcdbSynchronizer.js'); -const { GaqWorker } = require('./server/services/gaq/GaqWorker.js'); +const { gaqWorker } = require('./server/services/gaq/GaqWorker.js'); const { promises: fs } = require('fs'); const { MonAlisaClient } = require('./server/externalServicesSynchronization/monalisa/MonAlisaClient.js'); const https = require('https'); @@ -134,7 +134,6 @@ class BookkeepingApplication { } if (gaqConfig.enableRecalculation) { - const gaqWorker = new GaqWorker(); this.scheduledProcessesManager.schedule( () => gaqWorker.recalculateGaqSummaries(gaqConfig.batchSize), { diff --git a/lib/config/services.js b/lib/config/services.js index 2c9f941c6a..4c77a1c314 100644 --- a/lib/config/services.js +++ b/lib/config/services.js @@ -71,7 +71,7 @@ exports.services = { gaq: { enableRecalculation: process.env?.GAQ_ENABLE_RECALCULATION?.toLowerCase() === 'true' || true, - recalculationPeriod: Number(process.env?.GAQ_RECALCULATION_PERIOD) || 60 * 1000, // 1m in milliseconds + recalculationPeriod: Number(process.env?.GAQ_RECALCULATION_PERIOD) || 10 * 1000, // 10s in milliseconds batchSize: Number(process.env?.GAQ_RECALCULATION_BATCH_SIZE) || 1, }, }; diff --git a/lib/server/services/gaq/GaqWorker.js b/lib/server/services/gaq/GaqWorker.js index 1fa01920c0..a89671aba6 100644 --- a/lib/server/services/gaq/GaqWorker.js +++ b/lib/server/services/gaq/GaqWorker.js @@ -10,15 +10,33 @@ class GaqWorker { */ constructor() { this._logger = LogManager.getLogger(GaqWorker.name); + this._isPaused = false; + this._isSynchronizing = false; } /** - * Process pending GAQ summary invalidations. Skips if a previous call is still in progress. + * Pause the worker so it skips processing on the next scheduled calls + * @return {void} + */ + pause() { + this._isPaused = true; + } + + /** + * Resume the worker after a pause + * @return {void} + */ + resume() { + this._isPaused = false; + } + + /** + * Process pending GAQ summary invalidations. Skips if a previous call is still in progress or if paused. * @param {number} batchSize number of invalid summaries to recalculate * @return {Promise} promise */ async recalculateGaqSummaries(batchSize) { - if (this._isSynchronizing) { + if (this._isSynchronizing || this._isPaused) { return; } this._isSynchronizing = true; @@ -32,4 +50,6 @@ class GaqWorker { } } -exports.GaqWorker = GaqWorker; +const gaqWorker = new GaqWorker(); + +exports.gaqWorker = gaqWorker; From 10fac20a9189a4a5685ee4dcd4905fec12e1cd8f Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Wed, 1 Apr 2026 18:23:50 +0200 Subject: [PATCH 15/39] [O2B-1564] Add GAQ worker tests; pause worker in DB reset Adds tests that verify the worker removes an invalidation and adds a summary, upserts for an already present summary, batch processes correctly, and doesn't run concurrent recalculations. Update resetDatabaseContent to pause/resume the GAQ worker to avoid worker failures when the invalidation table is dropped. --- .../server/services/gaq/GaqSummary.test.js | 168 +++++++++++++++--- test/utilities/resetDatabaseContent.js | 5 + 2 files changed, 149 insertions(+), 24 deletions(-) diff --git a/test/lib/server/services/gaq/GaqSummary.test.js b/test/lib/server/services/gaq/GaqSummary.test.js index bffd5c5f05..ae95a7a140 100644 --- a/test/lib/server/services/gaq/GaqSummary.test.js +++ b/test/lib/server/services/gaq/GaqSummary.test.js @@ -12,11 +12,41 @@ */ const { expect } = require('chai'); +const sinon = require('sinon'); const { resetDatabaseContent } = require('../../../../utilities/resetDatabaseContent.js'); -const { repositories: { GaqSummaryInvalidationRepository } } = require('../../../../../lib/database'); +const { repositories: { GaqSummaryInvalidationRepository, GaqSummaryRepository } } = require('../../../../../lib/database'); const { qcFlagService } = require('../../../../../lib/server/services/qualityControlFlag/QcFlagService.js'); const { gaqDetectorService } = require('../../../../../lib/server/services/gaq/GaqDetectorsService.js'); const { updateRun } = require('../../../../../lib/server/services/run/updateRun.js'); +const { gaqWorker } = require('../../../../../lib/server/services/gaq/GaqWorker.js'); +const { gaqService } = require('../../../../../lib/server/services/gaq/GaqService.js'); + +/** + * Wait for a given number of milliseconds + * @param {number} ms milliseconds to wait + * @return {Promise} + */ +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Check whether an invalidation entry exists for a given data pass and run + * + * @param {number} expectedDataPassId + * @param {number} expectedRunNumber + * @param {boolean} toBePresent whether the invalidation is expected to be present + * + * @return {Promise} + */ +const expectInvalidation = async (expectedDataPassId, expectedRunNumber, toBePresent = true) => { + const invalidation = await GaqSummaryInvalidationRepository.findOne({ + where: { dataPassId: expectedDataPassId, runNumber: expectedRunNumber }, + }); + if (!toBePresent) { + expect(invalidation, `Expected no invalidation for dataPassId=${expectedDataPassId} runNumber=${expectedRunNumber}`).to.be.null; + } else { + expect(invalidation, `Expected invalidation for dataPassId=${expectedDataPassId} runNumber=${expectedRunNumber}`).to.not.be.null; + } +}; module.exports = () => { // Test resets the database before running and clears the invalidation table between each case @@ -28,27 +58,10 @@ module.exports = () => { const dataPassId = 4; // LHC22a_apass2 const runNumber = 56; - /** - * Check whether an invalidation entry exists for a given data pass and run - * - * @param {number} expectedDataPassId - * @param {number} expectedRunNumber - * @param {boolean} toBeNull - * - * @return {Promise} - */ - const expectInvalidation = async (expectedDataPassId, expectedRunNumber, toBeNull = false) => { - const invalidation = await GaqSummaryInvalidationRepository.findOne({ - where: { dataPassId: expectedDataPassId, runNumber: expectedRunNumber }, - }); - if (toBeNull) { - expect(invalidation, `Expected no invalidation for dataPassId=${expectedDataPassId} runNumber=${expectedRunNumber}`).to.be.null; - } else { - expect(invalidation, `Expected invalidation for dataPassId=${expectedDataPassId} runNumber=${expectedRunNumber}`).to.not.be.null; - } - }; - - describe("GAQ Summary Invalidation", async () => { + describe('GAQ Summary Invalidation', () => { + before(() => gaqWorker.pause()); + after(() => gaqWorker.resume()); + afterEach(async () => { await GaqSummaryInvalidationRepository.removeAll({ truncate: true }); }); @@ -75,7 +88,7 @@ module.exports = () => { // Verify again to check that no new invalidation is created when the flag is already verified await qcFlagService.verifyFlag({ flagId }, relations); - await expectInvalidation(dataPassId, 100, true); + await expectInvalidation(dataPassId, 100, false); }); it('should invalidate GAQ summary when a QC flag is deleted for a data pass', async () => { @@ -89,7 +102,6 @@ module.exports = () => { await qcFlagService.deleteAllForDataPass(dataPassId); await expectInvalidation(dataPassId, 100); - await expectInvalidation(dataPassId, 105); }); it('should invalidate GAQ summary when GAQ detectors are explicitly set for a data pass and run', async () => { @@ -118,4 +130,112 @@ module.exports = () => { await expectInvalidation(dataPassId, runNumber); }); }); + + describe('GAQ Worker', () => { + beforeEach(async () => { + await resetDatabaseContent(); + }); + + after(() => gaqWorker.pause()); + + it('should process invalidations and update the summary', async () => { + const workerDataPassId = 1; + const workerRunNumber = 107; + + await qcFlagService.create( + [{ from: null, to: null, flagTypeId: 3 }], + { runNumber: workerRunNumber, detectorIdentifier: { detectorId: 1 }, dataPassIdentifier: { id: workerDataPassId } }, + relations, + ); + + // confirm that the invalidation is made + await expectInvalidation(workerDataPassId, workerRunNumber); + + // wait at least 11s, default recalculation period is 10s, for the worker to process the invalidation + await sleep(11000); + + await expectInvalidation(workerDataPassId, workerRunNumber, false); + const summary = await GaqSummaryRepository.findOne({ where: { dataPassId: workerDataPassId, runNumber: workerRunNumber } }); + expect(summary).to.not.be.null; + }); + + it('should only upsert an existing summary row rather than creating a duplicate', async () => { + const workerDataPassId = 1; + const workerRunNumber = 107; + + await gaqService.calculateAndStoreGaqSummary(workerDataPassId, workerRunNumber); + const firstSummary = await GaqSummaryRepository.findOne({ where: { dataPassId: workerDataPassId, runNumber: workerRunNumber } }); + expect(firstSummary).to.not.be.null; + + // Trigger an invalidation + await qcFlagService.create( + [{ from: null, to: null, flagTypeId: 3 }], + { runNumber: workerRunNumber, detectorIdentifier: { detectorId: 1 }, dataPassIdentifier: { id: workerDataPassId } }, + relations, + ); + await expectInvalidation(workerDataPassId, workerRunNumber); + + // wait 11s for the worker to process + await sleep(11000); + + await expectInvalidation(workerDataPassId, workerRunNumber, false); + + // confirm only one summary row exists (upsert, not duplicate) + const summaries = await GaqSummaryRepository.findAll({ where: { dataPassId: workerDataPassId, runNumber: workerRunNumber } }); + expect(summaries).to.have.lengthOf(1); + }); + + it('should process multiple invalidations in a single batch', async () => { + // Create invalidations for two different runs in data pass 1 + await qcFlagService.create( + [{ from: null, to: null, flagTypeId: 3 }], + { runNumber: 106, detectorIdentifier: { detectorId: 1 }, dataPassIdentifier: { id: 1 } }, + relations, + ); + await qcFlagService.create( + [{ from: null, to: null, flagTypeId: 3 }], + { runNumber: 107, detectorIdentifier: { detectorId: 1 }, dataPassIdentifier: { id: 1 } }, + relations, + ); + + await expectInvalidation(1, 106); + await expectInvalidation(1, 107); + + // Manually call the worker with batchSize=2 to process both in one go + await gaqWorker.recalculateGaqSummaries(2); + + await expectInvalidation(1, 106, false); + await expectInvalidation(1, 107, false); + + const summary106 = await GaqSummaryRepository.findOne({ where: { dataPassId: 1, runNumber: 106 } }); + const summary107 = await GaqSummaryRepository.findOne({ where: { dataPassId: 1, runNumber: 107 } }); + expect(summary106).to.not.be.null; + expect(summary107).to.not.be.null; + }); + + it('should skip processing if a previous call is still in progress', async () => { + // Stub gaqService to be slow so the first call blocks + let resolveFirst; + const slowPromise = new Promise((resolve) => { resolveFirst = resolve; }); + const stub = sinon.stub(gaqService, 'popNInvalidSummaryAndRecalculate').returns(slowPromise); + + try { + // First call — will be held open by the slow stub + const firstCall = gaqWorker.recalculateGaqSummaries(1); + + // Second call — should be skipped because _isSynchronizing is true + await gaqWorker.recalculateGaqSummaries(1); + + // Stub should only have been called once + expect(stub.callCount).to.equal(1); + + // Release the first call + resolveFirst(); + await firstCall; + } finally { + sinon.restore(); + } + }); + + }); }; diff --git a/test/utilities/resetDatabaseContent.js b/test/utilities/resetDatabaseContent.js index 9e04ebddab..704a986219 100644 --- a/test/utilities/resetDatabaseContent.js +++ b/test/utilities/resetDatabaseContent.js @@ -12,9 +12,14 @@ */ const { database } = require('../../lib/application.js'); +const { gaqWorker } = require('../../lib/server/services/gaq/GaqWorker.js'); exports.resetDatabaseContent = async () => { + // Pause GAQ worker when resetDatabaseContent() runs in between tests in the different test suites + // Otherwise, worker fails as the invalidation table is DROPPED. This avoids an ERROR message appearing in test logs even if the suite passed + gaqWorker.pause(); await database.dropAllTables(); await database.migrate(); await database.seed(); + gaqWorker.resume(); }; From 6aa96175346fb2e440e3f63be6a19dd347bc83ec Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Wed, 1 Apr 2026 18:53:43 +0200 Subject: [PATCH 16/39] [O2B-1564] Add GaqService tests and register suite GaqService summary recalculation and invalidation processing functions tested directly circumventing worker. Tests verify correct computed fields, summary upsert, behaviour when no coverage exists, and batched invalidation processing. --- .../server/services/gaq/GaqService.test.js | 118 ++++++++++++++++++ test/lib/server/services/gaq/index.js | 2 + 2 files changed, 120 insertions(+) create mode 100644 test/lib/server/services/gaq/GaqService.test.js diff --git a/test/lib/server/services/gaq/GaqService.test.js b/test/lib/server/services/gaq/GaqService.test.js new file mode 100644 index 0000000000..3857f41397 --- /dev/null +++ b/test/lib/server/services/gaq/GaqService.test.js @@ -0,0 +1,118 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const { expect } = require('chai'); +const { resetDatabaseContent } = require('../../../../utilities/resetDatabaseContent.js'); +const { repositories: { GaqSummaryRepository, GaqSummaryInvalidationRepository } } = require('../../../../../lib/database'); +const { gaqService } = require('../../../../../lib/server/services/gaq/GaqService.js'); + +/** + * Find the GAQ summary row for a given data pass and run + * @param {number} dataPassId data pass id + * @param {number} runNumber run number + * @return {Promise} + */ +const findSummary = (dataPassId, runNumber) => GaqSummaryRepository.findOne({ where: { dataPassId, runNumber } }); + +/** + * Insert an invalidation entry + * @param {number} dataPassId data pass id + * @param {number} runNumber run number + * @param {string} createdAt ISO timestamp string + * @return {Promise} + */ +const insertInvalidation = (dataPassId, runNumber, createdAt) => + GaqSummaryInvalidationRepository.insert({ dataPassId, runNumber, createdAt: new Date(createdAt) }); + +// Tests for GaqService are split between QcFlagService.test.js and GaqSummary.test.js +// GaqService.test.js (this file) focuses on the summary recalculation and invalidation processing logic + +module.exports = () => { + before(resetDatabaseContent); + + // Data pass 1 (LHC22b_apass1), run 107 has GAQ detectors CPV (1) and ACO (2) seeded + // and has seeded QC flags, so a summary can always be computed + const dataPassId = 1; + const runNumber = 107; + + describe('calculateAndStoreGaqSummary', () => { + afterEach(async () => { + await GaqSummaryRepository.removeAll({ where: { dataPassId, runNumber } }); + }); + + it('should compute and store a summary row with correct values', async () => { + await gaqService.calculateAndStoreGaqSummary(dataPassId, runNumber); + + const summary = await findSummary(dataPassId, runNumber); + expect(summary).to.not.be.null; + expect(summary.dataPassId).to.equal(dataPassId); + expect(summary.runNumber).to.equal(runNumber); + expect(summary.badRunCoverage).to.equal(0); + expect(summary.explicitlyNotBadRunCoverage).to.equal(0.759654); + expect(summary.mcReproducibleCoverage).to.equal(0.240346); + expect(summary.missingVerificationsCount).to.equal(3); + expect(summary.undefinedQualityPeriodsCount).to.equal(0); + }); + + it('should upsert when a summary already exists', async () => { + await gaqService.calculateAndStoreGaqSummary(dataPassId, runNumber); + await gaqService.calculateAndStoreGaqSummary(dataPassId, runNumber); + + const rows = await GaqSummaryRepository.findAll({ where: { dataPassId, runNumber } }); + expect(rows).to.have.lengthOf(1); + }); + + it('should not store a summary when there is no coverage data for the run', async () => { + // Run 49 has no QC flags seeded for data pass 1 + await gaqService.calculateAndStoreGaqSummary(dataPassId, 49); + + const summary = await findSummary(dataPassId, 49); + expect(summary).to.be.null; + }); + }); + + describe('popNInvalidSummaryAndRecalculate', () => { + beforeEach(async () => { + await GaqSummaryInvalidationRepository.removeAll({ truncate: true }); + }); + + it('should do nothing when the invalidation table is empty', async () => { + await gaqService.popNInvalidSummaryAndRecalculate(5); + // No error thrown and nothing written + const summary = await findSummary(dataPassId, runNumber); + expect(summary).to.be.null; + }); + + it('should process exactly N invalidations ordered by createdAt', async () => { + // Insert invalidations for runs 106 and 107 in data pass 1 with different timestamps + await insertInvalidation(dataPassId, 106, '2024-01-01 10:00:00'); + await insertInvalidation(dataPassId, 107, '2024-01-01 11:00:00'); + + // Process only 1 — should pick run 106 (oldest) + await gaqService.popNInvalidSummaryAndRecalculate(1); + + const remaining = await GaqSummaryInvalidationRepository.findOne({ where: { dataPassId, runNumber: 107 } }); + expect(remaining).to.not.be.null; + }); + + it('should process all invalidations when batchSize covers them all', async () => { + await insertInvalidation(dataPassId, 106, '2024-01-01 10:00:00'); + await insertInvalidation(dataPassId, 107, '2024-01-01 11:00:00'); + + await gaqService.popNInvalidSummaryAndRecalculate(10); + + const count = await GaqSummaryInvalidationRepository.count({ where: { dataPassId } }); + expect(count).to.equal(0); + }); + }); +}; diff --git a/test/lib/server/services/gaq/index.js b/test/lib/server/services/gaq/index.js index 93e10b2ac6..32200b17d0 100644 --- a/test/lib/server/services/gaq/index.js +++ b/test/lib/server/services/gaq/index.js @@ -12,9 +12,11 @@ */ const GaqDetectorServiceSuite = require('./GaqDetectorService.test.js'); +const GaqServiceSuite = require('./GaqService.test.js'); const GaqSummarySuite = require('./GaqSummary.test.js'); module.exports = () => { describe('GaqDetectorService', GaqDetectorServiceSuite); + describe('GaqService', GaqServiceSuite); describe('GaqSummary', GaqSummarySuite); }; From c2ccbd476f83ce5ffb60097db8d1f373c61812b5 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Fri, 17 Apr 2026 15:52:40 +0200 Subject: [PATCH 17/39] Add calculationFailed to GAQ summaries Add a calculation_failed boolean column to gaq_summaries. This allows us to know whether a summary has been attempted to be calculated but unsuccessful due to limited data etc. --- lib/database/adapters/GaqSummaryAdapter.js | 2 ++ .../v1/20260223120000-create-gaq-summary-tables.js | 5 +++++ lib/database/models/gaqSummary.js | 3 +++ 3 files changed, 10 insertions(+) diff --git a/lib/database/adapters/GaqSummaryAdapter.js b/lib/database/adapters/GaqSummaryAdapter.js index 77ff939d69..21cd7b7004 100644 --- a/lib/database/adapters/GaqSummaryAdapter.js +++ b/lib/database/adapters/GaqSummaryAdapter.js @@ -37,6 +37,7 @@ class GaqSummaryAdapter { mcReproducibleCoverage, missingVerificationsCount, undefinedQualityPeriodsCount, + calculationFailed, createdAt, updatedAt, } = databaseObject; @@ -49,6 +50,7 @@ class GaqSummaryAdapter { mcReproducibleCoverage, missingVerificationsCount, undefinedQualityPeriodsCount, + calculationFailed, createdAt, updatedAt, }; diff --git a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js index b734f32e32..3a787b02e7 100644 --- a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js +++ b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js @@ -42,6 +42,11 @@ module.exports = { type: Sequelize.INTEGER, allowNull: false, }, + calculation_failed: { + type: Sequelize.BOOLEAN, + allowNull: false, + defaultValue: false, + }, created_at: { type: Sequelize.DATE(3), allowNull: false, diff --git a/lib/database/models/gaqSummary.js b/lib/database/models/gaqSummary.js index 480a9a4943..aab9a04dfd 100644 --- a/lib/database/models/gaqSummary.js +++ b/lib/database/models/gaqSummary.js @@ -36,6 +36,9 @@ module.exports = (sequelize) => { undefinedQualityPeriodsCount: { type: Sequelize.INTEGER, }, + calculationFailed: { + type: Sequelize.BOOLEAN, + }, }, { tableName: 'gaq_summaries' }); GaqSummary.removeAttribute('id'); From 35112e5e34b7c8f75240e086dd1cd100abc395da Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Fri, 17 Apr 2026 16:15:47 +0200 Subject: [PATCH 18/39] [O2B-1545] Make GAQ summary columns nullable Remove not-null constraints from several GAQ summary columns in the migration to allow NULL when values are unavailable. --- .../v1/20260223120000-create-gaq-summary-tables.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js index 3a787b02e7..4540a01c4a 100644 --- a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js +++ b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js @@ -24,23 +24,18 @@ module.exports = { }, bad_run_coverage: { type: Sequelize.FLOAT, - allowNull: false, }, explicitly_not_bad_run_coverage: { type: Sequelize.FLOAT, - allowNull: false, }, mc_reproducible_coverage: { type: Sequelize.FLOAT, - allowNull: false, }, missing_verifications_count: { type: Sequelize.INTEGER, - allowNull: false, }, undefined_quality_periods_count: { type: Sequelize.INTEGER, - allowNull: false, }, calculation_failed: { type: Sequelize.BOOLEAN, From 83ddf1d3ab908de5254ea51f3b3a715c610f12c2 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Fri, 17 Apr 2026 16:35:00 +0200 Subject: [PATCH 19/39] [O2B-1564] Persist GAQ calculation failure Stop returning early when summary is missing; always upsert a GaqSummary record and include a calculationFailed boolean. This ensures failed calculations are persisted for monitoring instead of being silently skipped. --- lib/server/services/gaq/GaqService.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/server/services/gaq/GaqService.js b/lib/server/services/gaq/GaqService.js index 79997ad06c..d7b9de7676 100644 --- a/lib/server/services/gaq/GaqService.js +++ b/lib/server/services/gaq/GaqService.js @@ -162,10 +162,7 @@ class GaqService { */ async calculateAndStoreGaqSummary(dataPassId, runNumber) { const summary = await this._computeSummary(dataPassId, runNumber); - if (!summary) { - return; - } - await GaqSummaryRepository.upsert({ dataPassId, runNumber, ...summary }); + await GaqSummaryRepository.upsert({ dataPassId, runNumber, ...summary, calculationFailed: summary === null }); } /** From 00c1e2bb2e981b9b1adcf8cc4d27d7c99ecce01e Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 28 Apr 2026 22:46:46 +0200 Subject: [PATCH 20/39] [O2B-1564] Assert calculationFailed in created summary Update test to expect that when there is no coverage data for a run, a GAQ summary is stored with calculationFailed set to true. --- test/lib/server/services/gaq/GaqService.test.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/lib/server/services/gaq/GaqService.test.js b/test/lib/server/services/gaq/GaqService.test.js index 3857f41397..e2f66d3e3d 100644 --- a/test/lib/server/services/gaq/GaqService.test.js +++ b/test/lib/server/services/gaq/GaqService.test.js @@ -72,12 +72,13 @@ module.exports = () => { expect(rows).to.have.lengthOf(1); }); - it('should not store a summary when there is no coverage data for the run', async () => { + it('should store a summary with calculation failed when there is no coverage data for the run', async () => { // Run 49 has no QC flags seeded for data pass 1 await gaqService.calculateAndStoreGaqSummary(dataPassId, 49); const summary = await findSummary(dataPassId, 49); - expect(summary).to.be.null; + expect(summary).to.not.be.null; + expect(summary.calculationFailed).to.be.true; }); }); From 5059082f89dae13ed149d910b413e0757e87c0b3 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Mon, 4 May 2026 14:20:48 +0200 Subject: [PATCH 21/39] [O2B-1545] Add invalidatedAt to gaq_summaries, remove invalidation table Store invalidation timestamp directly on gaq_summaries instead of a separate gaq_summary_invalidations table. --- lib/database/adapters/GaqSummaryAdapter.js | 2 ++ ...0260223120000-create-gaq-summary-tables.js | 33 ++----------------- lib/database/models/gaqSummary.js | 3 ++ 3 files changed, 8 insertions(+), 30 deletions(-) diff --git a/lib/database/adapters/GaqSummaryAdapter.js b/lib/database/adapters/GaqSummaryAdapter.js index 21cd7b7004..262fceb5be 100644 --- a/lib/database/adapters/GaqSummaryAdapter.js +++ b/lib/database/adapters/GaqSummaryAdapter.js @@ -38,6 +38,7 @@ class GaqSummaryAdapter { missingVerificationsCount, undefinedQualityPeriodsCount, calculationFailed, + invalidatedAt, createdAt, updatedAt, } = databaseObject; @@ -51,6 +52,7 @@ class GaqSummaryAdapter { missingVerificationsCount, undefinedQualityPeriodsCount, calculationFailed, + invalidatedAt, createdAt, updatedAt, }; diff --git a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js index 4540a01c4a..05b2693702 100644 --- a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js +++ b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js @@ -42,36 +42,10 @@ module.exports = { allowNull: false, defaultValue: false, }, - created_at: { + invalidated_at: { type: Sequelize.DATE(3), - allowNull: false, - defaultValue: Sequelize.literal('CURRENT_TIMESTAMP(3)'), - }, - updated_at: { - type: Sequelize.DATE(3), - allowNull: false, - defaultValue: Sequelize.literal('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'), - }, - }, { transaction }); - - await queryInterface.createTable('gaq_summary_invalidations', { - data_pass_id: { - type: Sequelize.INTEGER, - primaryKey: true, - allowNull: false, - references: { - model: 'data_passes', - key: 'id', - }, - }, - run_number: { - type: Sequelize.INTEGER, - primaryKey: true, - allowNull: false, - references: { - model: 'runs', - key: 'run_number', - }, + allowNull: true, + defaultValue: null, }, created_at: { type: Sequelize.DATE(3), @@ -87,7 +61,6 @@ module.exports = { }), down: async (queryInterface) => queryInterface.sequelize.transaction(async (transaction) => { - await queryInterface.dropTable('gaq_summary_invalidations', { transaction }); await queryInterface.dropTable('gaq_summaries', { transaction }); }), }; diff --git a/lib/database/models/gaqSummary.js b/lib/database/models/gaqSummary.js index aab9a04dfd..97099f256d 100644 --- a/lib/database/models/gaqSummary.js +++ b/lib/database/models/gaqSummary.js @@ -39,6 +39,9 @@ module.exports = (sequelize) => { calculationFailed: { type: Sequelize.BOOLEAN, }, + invalidatedAt: { + type: Sequelize.DATE(3), + }, }, { tableName: 'gaq_summaries' }); GaqSummary.removeAttribute('id'); From 9893667ba7ccb7629c8c66d1b77cec5fb11c580d Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Mon, 4 May 2026 14:23:39 +0200 Subject: [PATCH 22/39] [O2B-1545] Rename calculationFailed to notComputable --- lib/database/adapters/GaqSummaryAdapter.js | 4 ++-- .../migrations/v1/20260223120000-create-gaq-summary-tables.js | 2 +- lib/database/models/gaqSummary.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/database/adapters/GaqSummaryAdapter.js b/lib/database/adapters/GaqSummaryAdapter.js index 262fceb5be..53b6baf61a 100644 --- a/lib/database/adapters/GaqSummaryAdapter.js +++ b/lib/database/adapters/GaqSummaryAdapter.js @@ -37,7 +37,7 @@ class GaqSummaryAdapter { mcReproducibleCoverage, missingVerificationsCount, undefinedQualityPeriodsCount, - calculationFailed, + notComputable, invalidatedAt, createdAt, updatedAt, @@ -51,7 +51,7 @@ class GaqSummaryAdapter { mcReproducibleCoverage, missingVerificationsCount, undefinedQualityPeriodsCount, - calculationFailed, + notComputable, invalidatedAt, createdAt, updatedAt, diff --git a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js index 05b2693702..9366d050ce 100644 --- a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js +++ b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js @@ -37,7 +37,7 @@ module.exports = { undefined_quality_periods_count: { type: Sequelize.INTEGER, }, - calculation_failed: { + not_computable: { type: Sequelize.BOOLEAN, allowNull: false, defaultValue: false, diff --git a/lib/database/models/gaqSummary.js b/lib/database/models/gaqSummary.js index 97099f256d..efa76e04fd 100644 --- a/lib/database/models/gaqSummary.js +++ b/lib/database/models/gaqSummary.js @@ -36,7 +36,7 @@ module.exports = (sequelize) => { undefinedQualityPeriodsCount: { type: Sequelize.INTEGER, }, - calculationFailed: { + notComputable: { type: Sequelize.BOOLEAN, }, invalidatedAt: { From 47449a7da2403962c4e7417076d06807f38ad453 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Mon, 4 May 2026 14:31:43 +0200 Subject: [PATCH 23/39] [O2B-1545] Remove unused GaqSummaryInvalidation files and imports --- .../adapters/GaqSummaryInvalidationAdapter.js | 48 ------------------- lib/database/adapters/index.js | 3 -- lib/database/models/gaqSummaryInvalidation.js | 34 ------------- lib/database/models/index.js | 2 - .../GaqSummaryInvalidationRepository.js | 33 ------------- lib/database/repositories/index.js | 2 - 6 files changed, 122 deletions(-) delete mode 100644 lib/database/adapters/GaqSummaryInvalidationAdapter.js delete mode 100644 lib/database/models/gaqSummaryInvalidation.js delete mode 100644 lib/database/repositories/GaqSummaryInvalidationRepository.js diff --git a/lib/database/adapters/GaqSummaryInvalidationAdapter.js b/lib/database/adapters/GaqSummaryInvalidationAdapter.js deleted file mode 100644 index dbd76df3a8..0000000000 --- a/lib/database/adapters/GaqSummaryInvalidationAdapter.js +++ /dev/null @@ -1,48 +0,0 @@ -/** - * @license - * Copyright CERN and copyright holders of ALICE O2. This software is - * distributed under the terms of the GNU General Public License v3 (GPL - * Version 3), copied verbatim in the file "COPYING". - * - * See http://alice-o2.web.cern.ch/license for full licensing information. - * - * In applying this license CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization - * or submit itself to any jurisdiction. - */ - -/** - * GaqSummaryInvalidationAdapter - */ -class GaqSummaryInvalidationAdapter { - /** - * Constructor - */ - constructor() { - this.toEntity = this.toEntity.bind(this); - } - - /** - * Converts the given database object to an entity object. - * - * @param {SequelizeGaqSummaryInvalidation} databaseObject Object to convert. - * @returns {GaqSummaryInvalidation} Converted entity object. - */ - toEntity(databaseObject) { - const { - dataPassId, - runNumber, - createdAt, - updatedAt, - } = databaseObject; - - return { - dataPassId, - runNumber, - createdAt, - updatedAt, - }; - } -} - -module.exports = { GaqSummaryInvalidationAdapter }; diff --git a/lib/database/adapters/index.js b/lib/database/adapters/index.js index 97d22b927f..9c858e1bc2 100644 --- a/lib/database/adapters/index.js +++ b/lib/database/adapters/index.js @@ -28,7 +28,6 @@ const FlpRoleAdapter = require('./FlpRoleAdapter'); const { HostAdapter } = require('./HostAdapter.js'); const { GaqDetectorAdapter } = require('./GaqDetectorAdapter.js'); const { GaqSummaryAdapter } = require('./GaqSummaryAdapter.js'); -const { GaqSummaryInvalidationAdapter } = require('./GaqSummaryInvalidationAdapter.js'); const { LhcFillAdapter } = require('./LhcFillAdapter.js'); const { LhcFillStatisticsAdapter } = require('./LhcFillStatisticsAdapter.js'); const LhcPeriodAdapter = require('./LhcPeriodAdapter'); @@ -66,7 +65,6 @@ const eorReasonAdapter = new EorReasonAdapter(); const flpRoleAdapter = new FlpRoleAdapter(); const gaqDetectorAdapter = new GaqDetectorAdapter(); const gaqSummaryAdapter = new GaqSummaryAdapter(); -const gaqSummaryInvalidationAdapter = new GaqSummaryInvalidationAdapter(); const hostAdapter = new HostAdapter(); const lhcFillAdapter = new LhcFillAdapter(); const lhcFillStatisticsAdapter = new LhcFillStatisticsAdapter(); @@ -164,7 +162,6 @@ module.exports = { flpRoleAdapter, gaqDetectorAdapter, gaqSummaryAdapter, - gaqSummaryInvalidationAdapter, hostAdapter, lhcFillAdapter, lhcFillStatisticsAdapter, diff --git a/lib/database/models/gaqSummaryInvalidation.js b/lib/database/models/gaqSummaryInvalidation.js deleted file mode 100644 index 3c0944b9bc..0000000000 --- a/lib/database/models/gaqSummaryInvalidation.js +++ /dev/null @@ -1,34 +0,0 @@ -/** - * @license - * Copyright CERN and copyright holders of ALICE O2. This software is - * distributed under the terms of the GNU General Public License v3 (GPL - * Version 3), copied verbatim in the file "COPYING". - * - * See http://alice-o2.web.cern.ch/license for full licensing information. - * - * In applying this license CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization - * or submit itself to any jurisdiction. - */ - -module.exports = (sequelize) => { - const Sequelize = require('sequelize'); - - const GaqSummaryInvalidation = sequelize.define('GaqSummaryInvalidation', { - dataPassId: { - type: Sequelize.INTEGER, - }, - runNumber: { - type: Sequelize.INTEGER, - }, - }, { tableName: 'gaq_summary_invalidations' }); - - GaqSummaryInvalidation.removeAttribute('id'); - - GaqSummaryInvalidation.associate = (models) => { - GaqSummaryInvalidation.belongsTo(models.Run, { foreignKey: 'runNumber', as: 'run' }); - GaqSummaryInvalidation.belongsTo(models.DataPass, { foreignKey: 'dataPassId', as: 'dataPass' }); - }; - - return GaqSummaryInvalidation; -}; diff --git a/lib/database/models/index.js b/lib/database/models/index.js index 36736ce905..2549209c5b 100644 --- a/lib/database/models/index.js +++ b/lib/database/models/index.js @@ -28,7 +28,6 @@ const EpnRoleSession = require('./epnrolesession'); const FlpRole = require('./flprole'); const GaqDetector = require('./gaqDetector.js'); const GaqSummary = require('./gaqSummary.js'); -const GaqSummaryInvalidation = require('./gaqSummaryInvalidation.js'); const Host = require('./host.js'); const LhcFill = require('./lhcFill'); const LhcFillStatistics = require('./lhcFillStatistics.js'); @@ -69,7 +68,6 @@ module.exports = (sequelize) => { FlpRole: FlpRole(sequelize), GaqDetector: GaqDetector(sequelize), GaqSummary: GaqSummary(sequelize), - GaqSummaryInvalidation: GaqSummaryInvalidation(sequelize), Host: Host(sequelize), LhcFill: LhcFill(sequelize), LhcFillStatistics: LhcFillStatistics(sequelize), diff --git a/lib/database/repositories/GaqSummaryInvalidationRepository.js b/lib/database/repositories/GaqSummaryInvalidationRepository.js deleted file mode 100644 index c6608f9e83..0000000000 --- a/lib/database/repositories/GaqSummaryInvalidationRepository.js +++ /dev/null @@ -1,33 +0,0 @@ -/** - * @license - * Copyright CERN and copyright holders of ALICE O2. This software is - * distributed under the terms of the GNU General Public License v3 (GPL - * Version 3), copied verbatim in the file "COPYING". - * - * See http://alice-o2.web.cern.ch/license for full licensing information. - * - * In applying this license CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization - * or submit itself to any jurisdiction. - */ - -const { - models: { - GaqSummaryInvalidation, - }, -} = require('..'); -const Repository = require('./Repository'); - -/** - * GaqSummaryInvalidation repository - */ -class GaqSummaryInvalidationRepository extends Repository { - /** - * Creates a new `GaqSummaryInvalidationRepository` instance. - */ - constructor() { - super(GaqSummaryInvalidation); - } -} - -module.exports = new GaqSummaryInvalidationRepository(); diff --git a/lib/database/repositories/index.js b/lib/database/repositories/index.js index 417ccd5375..4be7600145 100644 --- a/lib/database/repositories/index.js +++ b/lib/database/repositories/index.js @@ -27,7 +27,6 @@ const EnvironmentRepository = require('./EnvironmentRepository'); const EorReasonRepository = require('./EorReasonRepository'); const FlpRoleRepository = require('./FlpRoleRepository'); const GaqDetectorRepository = require('./GaqDetectorRepository.js'); -const GaqSummaryInvalidationRepository = require('./GaqSummaryInvalidationRepository.js'); const GaqSummaryRepository = require('./GaqSummaryRepository.js'); const HostRepository = require('./HostRepository.js'); const LhcFillRepository = require('./LhcFillRepository'); @@ -72,7 +71,6 @@ module.exports = { EorReasonRepository, FlpRoleRepository, GaqDetectorRepository, - GaqSummaryInvalidationRepository, GaqSummaryRepository, HostRepository, LhcFillRepository, From 0b8b161e72b0787da3ce644529ce7b0bb5d7c1d9 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Mon, 4 May 2026 15:05:13 +0200 Subject: [PATCH 24/39] [O2B-1563] Use GaqSummaryRepository invalidation Replace usage of GaqSummaryInvalidationRepository with GaqSummaryRepository across services and tests. Upsert calls now set an invalidatedAt timestamp to mark GAQ summaries as invalidated. --- lib/server/services/gaq/GaqDetectorsService.js | 8 +++++--- .../services/qualityControlFlag/QcFlagService.js | 14 +++++++++----- lib/server/services/run/updateRun.js | 5 +++-- test/lib/server/services/gaq/GaqSummary.test.js | 14 +++++++------- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/lib/server/services/gaq/GaqDetectorsService.js b/lib/server/services/gaq/GaqDetectorsService.js index 6c19d1cd4a..4411acdefa 100644 --- a/lib/server/services/gaq/GaqDetectorsService.js +++ b/lib/server/services/gaq/GaqDetectorsService.js @@ -12,7 +12,7 @@ */ const { gaqDetectorAdapter, detectorAdapter } = require('../../../database/adapters'); -const { GaqDetectorRepository, RunRepository, DetectorRepository, GaqSummaryInvalidationRepository } = require('../../../database/repositories'); +const { GaqDetectorRepository, RunRepository, DetectorRepository, GaqSummaryRepository } = require('../../../database/repositories'); const { BadParameterError } = require('../../errors/BadParameterError'); const { dataSource } = require('../../../database/DataSource.js'); const { Op } = require('sequelize'); @@ -59,9 +59,10 @@ class GaqDetectorService { const createdEntries = await GaqDetectorRepository.insertAll(gaqEntries); // Invalidate GAQ summaries for all affected runs - await Promise.all(runNumbers.map((runNumber) => GaqSummaryInvalidationRepository.upsert({ + await Promise.all(runNumbers.map((runNumber) => GaqSummaryRepository.upsert({ dataPassId, runNumber, + invalidatedAt: new Date(), }))); return createdEntries.map(gaqDetectorAdapter.toEntity); @@ -110,9 +111,10 @@ class GaqDetectorService { const createdEntries = await GaqDetectorRepository.insertAll(gaqEntries); // Invalidate GAQ summaries for all affected runs - await Promise.all(runNumbers.map((runNumber) => GaqSummaryInvalidationRepository.upsert({ + await Promise.all(runNumbers.map((runNumber) => GaqSummaryRepository.upsert({ dataPassId, runNumber, + invalidatedAt: new Date(), }))); return createdEntries.map(gaqDetectorAdapter.toEntity); diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 3f0ca96d79..e3dd1a9dda 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -20,7 +20,7 @@ const { RunRepository, QcFlagVerificationRepository, QcFlagEffectivePeriodRepository, - GaqSummaryInvalidationRepository, + GaqSummaryRepository, }, } = require('../../../database/index.js'); const { dataSource } = require('../../../database/DataSource.js'); @@ -212,9 +212,10 @@ class QcFlagService { if (dataPass) { // Invalidate GAQ summary for the dataPass and runNumber of the created flag - await GaqSummaryInvalidationRepository.upsert({ + await GaqSummaryRepository.upsert({ dataPassId: dataPass.id, runNumber, + invalidatedAt: new Date(), }); } @@ -296,9 +297,10 @@ class QcFlagService { if (dataPassId) { // Invalidate GAQ summary for the dataPass and runNumber of the created flag - await GaqSummaryInvalidationRepository.upsert({ + await GaqSummaryRepository.upsert({ dataPassId: dataPassId, runNumber, + invalidatedAt: new Date(), }); } @@ -379,9 +381,10 @@ class QcFlagService { ); // Invalidate GAQ summary for the dataPass and all runNumbers of the deleted flags - await Promise.all(Array.from(runNumbers).map((runNumber) => GaqSummaryInvalidationRepository.upsert({ + await Promise.all(Array.from(runNumbers).map((runNumber) => GaqSummaryRepository.upsert({ dataPassId, runNumber, + invalidatedAt: new Date(), }))); return qcFlagIds.length; @@ -425,9 +428,10 @@ class QcFlagService { // Invalidate GAQ summary if it's the first verification if (dataPassId && (!qcFlag.verifications || qcFlag.verifications.length === 0)) { - await GaqSummaryInvalidationRepository.upsert({ + await GaqSummaryRepository.upsert({ dataPassId, runNumber: updatedQcFlag.runNumber, + invalidatedAt: new Date(), }); } diff --git a/lib/server/services/run/updateRun.js b/lib/server/services/run/updateRun.js index 98c7247c53..a7833ed107 100644 --- a/lib/server/services/run/updateRun.js +++ b/lib/server/services/run/updateRun.js @@ -12,7 +12,7 @@ */ const RunRepository = require('../../../database/repositories/RunRepository.js'); -const GaqSummaryInvalidationRepository = require('../../../database/repositories/GaqSummaryInvalidationRepository.js'); +const GaqSummaryRepository = require('../../../database/repositories/GaqSummaryRepository.js'); const DataPassRunRepository = require('../../../database/repositories/DataPassRunRepository.js'); const { getRunOrFail } = require('./getRunOrFail.js'); const { utilities: { TransactionHelper } } = require('../../../database'); @@ -220,9 +220,10 @@ exports.updateRun = async (identifier, payload, transaction) => { where: { runNumber: runModel.runNumber }, }); for (const { dataPassId } of dataPassRuns) { - await GaqSummaryInvalidationRepository.upsert({ + await GaqSummaryRepository.upsert({ dataPassId, runNumber: runModel.runNumber, + invalidatedAt: new Date(), }); } } diff --git a/test/lib/server/services/gaq/GaqSummary.test.js b/test/lib/server/services/gaq/GaqSummary.test.js index bffd5c5f05..b4f6ae920d 100644 --- a/test/lib/server/services/gaq/GaqSummary.test.js +++ b/test/lib/server/services/gaq/GaqSummary.test.js @@ -13,13 +13,12 @@ const { expect } = require('chai'); const { resetDatabaseContent } = require('../../../../utilities/resetDatabaseContent.js'); -const { repositories: { GaqSummaryInvalidationRepository } } = require('../../../../../lib/database'); +const { repositories: { GaqSummaryRepository } } = require('../../../../../lib/database'); const { qcFlagService } = require('../../../../../lib/server/services/qualityControlFlag/QcFlagService.js'); const { gaqDetectorService } = require('../../../../../lib/server/services/gaq/GaqDetectorsService.js'); const { updateRun } = require('../../../../../lib/server/services/run/updateRun.js'); module.exports = () => { - // Test resets the database before running and clears the invalidation table between each case before(async () => { await resetDatabaseContent(); }); @@ -38,19 +37,20 @@ module.exports = () => { * @return {Promise} */ const expectInvalidation = async (expectedDataPassId, expectedRunNumber, toBeNull = false) => { - const invalidation = await GaqSummaryInvalidationRepository.findOne({ + const summary = await GaqSummaryRepository.findOne({ where: { dataPassId: expectedDataPassId, runNumber: expectedRunNumber }, }); if (toBeNull) { - expect(invalidation, `Expected no invalidation for dataPassId=${expectedDataPassId} runNumber=${expectedRunNumber}`).to.be.null; + expect(summary?.invalidatedAt, `Expected no invalidation for dataPassId=${expectedDataPassId} runNumber=${expectedRunNumber}`).to.be.null; } else { - expect(invalidation, `Expected invalidation for dataPassId=${expectedDataPassId} runNumber=${expectedRunNumber}`).to.not.be.null; + expect(summary?.invalidatedAt, `Expected invalidation for dataPassId=${expectedDataPassId} runNumber=${expectedRunNumber}`).to.not.be.null; } }; describe("GAQ Summary Invalidation", async () => { + // Resetting the invalidated column between each case afterEach(async () => { - await GaqSummaryInvalidationRepository.removeAll({ truncate: true }); + await GaqSummaryRepository.updateAll({ invalidatedAt: null }, { where: {} }); }); it('should invalidate GAQ summary when a QC flag is created for a data pass', async () => { @@ -71,7 +71,7 @@ module.exports = () => { await expectInvalidation(dataPassId, 100); // Clear invalidation - await GaqSummaryInvalidationRepository.removeAll({ where: { dataPassId, runNumber: 100 } }); + await GaqSummaryRepository.updateAll({ invalidatedAt: null }, { where: { dataPassId, runNumber: 100 } }); // Verify again to check that no new invalidation is created when the flag is already verified await qcFlagService.verifyFlag({ flagId }, relations); From f4d2f0d724c82e3a75bf09a8370c1a9b7245cd88 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Mon, 4 May 2026 16:35:43 +0200 Subject: [PATCH 25/39] [O2B-1564] Use invalidatedAt on GaqSummary in background worker --- lib/server/services/gaq/GaqService.js | 13 ++++++++----- test/lib/server/services/gaq/GaqService.test.js | 15 ++++++++------- test/lib/server/services/gaq/GaqSummary.test.js | 12 ++++++------ 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/lib/server/services/gaq/GaqService.js b/lib/server/services/gaq/GaqService.js index d7b9de7676..b649141dad 100644 --- a/lib/server/services/gaq/GaqService.js +++ b/lib/server/services/gaq/GaqService.js @@ -28,7 +28,7 @@ */ const { getOneDataPassOrFail } = require('../dataPasses/getOneDataPassOrFail.js'); -const { QcFlagRepository, GaqSummaryRepository, GaqSummaryInvalidationRepository } = require('../../../database/repositories/index.js'); +const { QcFlagRepository, GaqSummaryRepository } = require('../../../database/repositories/index.js'); const { qcFlagAdapter } = require('../../../database/adapters/index.js'); const { Op } = require('sequelize'); const { QcSummarProperties } = require('../../../domain/enums/QcSummaryProperties.js'); @@ -162,7 +162,7 @@ class GaqService { */ async calculateAndStoreGaqSummary(dataPassId, runNumber) { const summary = await this._computeSummary(dataPassId, runNumber); - await GaqSummaryRepository.upsert({ dataPassId, runNumber, ...summary, calculationFailed: summary === null }); + await GaqSummaryRepository.upsert({ dataPassId, runNumber, ...summary, notComputable: summary === null, invalidatedAt: null }); } /** @@ -171,16 +171,19 @@ class GaqService { * @return {Promise} promise */ async popNInvalidSummaryAndRecalculate(batchSize = 1) { - const invalidCount = await GaqSummaryInvalidationRepository.count(); + const invalidCount = await GaqSummaryRepository.count({ where: { invalidatedAt: { [Op.not]: null } } }); const remaining = Math.min(batchSize, invalidCount); if (remaining === 0) { return; } await dataSource.transaction(async () => { for (let i = 0; i < remaining; i++) { - const invalidation = await GaqSummaryInvalidationRepository.removeOne({ where: {}, order: [['createdAt', 'ASC']] }); + const invalidation = await GaqSummaryRepository.findOne({ + where: { invalidatedAt: { [Op.not]: null } }, + order: [['invalidatedAt', 'ASC']], + }); if (!invalidation) { - break; + return; } const { dataPassId, runNumber } = invalidation; await this.calculateAndStoreGaqSummary(dataPassId, runNumber); diff --git a/test/lib/server/services/gaq/GaqService.test.js b/test/lib/server/services/gaq/GaqService.test.js index e2f66d3e3d..baf6cbfb0b 100644 --- a/test/lib/server/services/gaq/GaqService.test.js +++ b/test/lib/server/services/gaq/GaqService.test.js @@ -13,8 +13,9 @@ const { expect } = require('chai'); const { resetDatabaseContent } = require('../../../../utilities/resetDatabaseContent.js'); -const { repositories: { GaqSummaryRepository, GaqSummaryInvalidationRepository } } = require('../../../../../lib/database'); +const { repositories: { GaqSummaryRepository} } = require('../../../../../lib/database'); const { gaqService } = require('../../../../../lib/server/services/gaq/GaqService.js'); +const { Op } = require('sequelize'); /** * Find the GAQ summary row for a given data pass and run @@ -32,7 +33,7 @@ const findSummary = (dataPassId, runNumber) => GaqSummaryRepository.findOne({ wh * @return {Promise} */ const insertInvalidation = (dataPassId, runNumber, createdAt) => - GaqSummaryInvalidationRepository.insert({ dataPassId, runNumber, createdAt: new Date(createdAt) }); + GaqSummaryRepository.upsert({ dataPassId, runNumber, invalidatedAt: new Date(createdAt) }); // Tests for GaqService are split between QcFlagService.test.js and GaqSummary.test.js // GaqService.test.js (this file) focuses on the summary recalculation and invalidation processing logic @@ -72,19 +73,19 @@ module.exports = () => { expect(rows).to.have.lengthOf(1); }); - it('should store a summary with calculation failed when there is no coverage data for the run', async () => { + it('should store a summary with notComputable set to true when there is no coverage data for the run', async () => { // Run 49 has no QC flags seeded for data pass 1 await gaqService.calculateAndStoreGaqSummary(dataPassId, 49); const summary = await findSummary(dataPassId, 49); expect(summary).to.not.be.null; - expect(summary.calculationFailed).to.be.true; + expect(summary.notComputable).to.be.true; }); }); describe('popNInvalidSummaryAndRecalculate', () => { beforeEach(async () => { - await GaqSummaryInvalidationRepository.removeAll({ truncate: true }); + await GaqSummaryRepository.updateAll({ invalidatedAt: null }, { where: {} }); }); it('should do nothing when the invalidation table is empty', async () => { @@ -102,7 +103,7 @@ module.exports = () => { // Process only 1 — should pick run 106 (oldest) await gaqService.popNInvalidSummaryAndRecalculate(1); - const remaining = await GaqSummaryInvalidationRepository.findOne({ where: { dataPassId, runNumber: 107 } }); + const remaining = await GaqSummaryRepository.findOne({ where: { dataPassId, runNumber: 107, invalidatedAt: { [Op.not]: null } } }); expect(remaining).to.not.be.null; }); @@ -112,7 +113,7 @@ module.exports = () => { await gaqService.popNInvalidSummaryAndRecalculate(10); - const count = await GaqSummaryInvalidationRepository.count({ where: { dataPassId } }); + const count = await GaqSummaryRepository.count({ where: { dataPassId, invalidatedAt: { [Op.not]: null } } }); expect(count).to.equal(0); }); }); diff --git a/test/lib/server/services/gaq/GaqSummary.test.js b/test/lib/server/services/gaq/GaqSummary.test.js index b3e459682e..9ccc94d4dc 100644 --- a/test/lib/server/services/gaq/GaqSummary.test.js +++ b/test/lib/server/services/gaq/GaqSummary.test.js @@ -88,7 +88,7 @@ module.exports = () => { // Verify again to check that no new invalidation is created when the flag is already verified await qcFlagService.verifyFlag({ flagId }, relations); - await expectInvalidation(dataPassId, 100, false); + await expectInvalidation(dataPassId, 100, true); }); it('should invalidate GAQ summary when a QC flag is deleted for a data pass', async () => { @@ -154,9 +154,9 @@ module.exports = () => { // wait at least 11s, default recalculation period is 10s, for the worker to process the invalidation await sleep(11000); - await expectInvalidation(workerDataPassId, workerRunNumber, false); + await expectInvalidation(workerDataPassId, workerRunNumber, true); const summary = await GaqSummaryRepository.findOne({ where: { dataPassId: workerDataPassId, runNumber: workerRunNumber } }); - expect(summary).to.not.be.null; + expect(summary.badRunCoverage).to.not.be.null; }); it('should only upsert an existing summary row rather than creating a duplicate', async () => { @@ -178,7 +178,7 @@ module.exports = () => { // wait 11s for the worker to process await sleep(11000); - await expectInvalidation(workerDataPassId, workerRunNumber, false); + await expectInvalidation(workerDataPassId, workerRunNumber, true); // confirm only one summary row exists (upsert, not duplicate) const summaries = await GaqSummaryRepository.findAll({ where: { dataPassId: workerDataPassId, runNumber: workerRunNumber } }); @@ -204,8 +204,8 @@ module.exports = () => { // Manually call the worker with batchSize=2 to process both in one go await gaqWorker.recalculateGaqSummaries(2); - await expectInvalidation(1, 106, false); - await expectInvalidation(1, 107, false); + await expectInvalidation(1, 106, true); + await expectInvalidation(1, 107, true); const summary106 = await GaqSummaryRepository.findOne({ where: { dataPassId: 1, runNumber: 106 } }); const summary107 = await GaqSummaryRepository.findOne({ where: { dataPassId: 1, runNumber: 107 } }); From a0b44cc2e72d8de350ddefe02d019796b30886f4 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 5 May 2026 09:40:32 +0200 Subject: [PATCH 26/39] [O2B-1564] Fix GAQ recalculation config and default env vars --- docker-compose.dev.yml | 3 +++ docker-compose.test.yml | 3 +++ lib/config/services.js | 6 +++--- test/lib/server/services/gaq/GaqSummary.test.js | 7 +++---- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index f782a26d3d..f8b026a81a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -10,6 +10,9 @@ services: JWT_SECRET: BOOKKEEPING-DEV GRPC_INTERNAL_ORIGIN: '[::]:4001' GRPC_AUTHENTICATED_ORIGIN: '[::]:4002' + GAQ_ENABLE_RECALCULATION: "True" + GAQ_RECALCULATION_PERIOD: 10000 + GAQ_RECALCULATION_BATCH_SIZE: 20 ports: - "4000:4000" - "4001:4001" diff --git a/docker-compose.test.yml b/docker-compose.test.yml index b43a5c8ef9..3d4bce9081 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -10,6 +10,9 @@ services: JWT_SECRET: BOOKKEEPING-TEST-SUITE PAGE_ITEMS_LIMIT: 100 CCDB_SYNCHRONIZATION_PERIOD: 3153600000000 # 100y in milliseconds, to be sure all the runs are included when testing sync + GAQ_ENABLE_RECALCULATION: "True" + GAQ_RECALCULATION_PERIOD: 1000 + GAQ_RECALCULATION_BATCH_SIZE: 1 restart: "no" database: diff --git a/lib/config/services.js b/lib/config/services.js index 4c77a1c314..e2a548565b 100644 --- a/lib/config/services.js +++ b/lib/config/services.js @@ -70,8 +70,8 @@ exports.services = { }, gaq: { - enableRecalculation: process.env?.GAQ_ENABLE_RECALCULATION?.toLowerCase() === 'true' || true, - recalculationPeriod: Number(process.env?.GAQ_RECALCULATION_PERIOD) || 10 * 1000, // 10s in milliseconds - batchSize: Number(process.env?.GAQ_RECALCULATION_BATCH_SIZE) || 1, + enableRecalculation: process.env?.GAQ_ENABLE_RECALCULATION?.toLowerCase() === 'true', + recalculationPeriod: Number(process.env?.GAQ_RECALCULATION_PERIOD) || 30 * 1000, // 30s in milliseconds + batchSize: Number(process.env?.GAQ_RECALCULATION_BATCH_SIZE) || 10, }, }; diff --git a/test/lib/server/services/gaq/GaqSummary.test.js b/test/lib/server/services/gaq/GaqSummary.test.js index 9ccc94d4dc..704fac682e 100644 --- a/test/lib/server/services/gaq/GaqSummary.test.js +++ b/test/lib/server/services/gaq/GaqSummary.test.js @@ -151,8 +151,8 @@ module.exports = () => { // confirm that the invalidation is made await expectInvalidation(workerDataPassId, workerRunNumber); - // wait at least 11s, default recalculation period is 10s, for the worker to process the invalidation - await sleep(11000); + // wait at least 2s, recalculation period is 1s in test env, for the worker to process the invalidation + await sleep(2000); await expectInvalidation(workerDataPassId, workerRunNumber, true); const summary = await GaqSummaryRepository.findOne({ where: { dataPassId: workerDataPassId, runNumber: workerRunNumber } }); @@ -175,8 +175,7 @@ module.exports = () => { ); await expectInvalidation(workerDataPassId, workerRunNumber); - // wait 11s for the worker to process - await sleep(11000); + await sleep(2000); await expectInvalidation(workerDataPassId, workerRunNumber, true); From 8bc1d4168baf27b538a02b632fd870459534e046 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 5 May 2026 10:08:58 +0200 Subject: [PATCH 27/39] [O2B-1564] Process invalid GAQ summaries concurrently in batches Replace iterative invalidation pop with a single query that fetches up to batchSize invalidated GAQ summaries and processes them concurrently with a Promise.all. --- lib/server/services/gaq/GaqService.js | 29 ++++++++++----------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/lib/server/services/gaq/GaqService.js b/lib/server/services/gaq/GaqService.js index b649141dad..cee14a5c19 100644 --- a/lib/server/services/gaq/GaqService.js +++ b/lib/server/services/gaq/GaqService.js @@ -32,7 +32,6 @@ const { QcFlagRepository, GaqSummaryRepository } = require('../../../database/re const { qcFlagAdapter } = require('../../../database/adapters/index.js'); const { Op } = require('sequelize'); const { QcSummarProperties } = require('../../../domain/enums/QcSummaryProperties.js'); -const { dataSource } = require('../../../database/DataSource.js'); const { LogManager } = require('@aliceo2/web-ui'); /** @@ -171,24 +170,18 @@ class GaqService { * @return {Promise} promise */ async popNInvalidSummaryAndRecalculate(batchSize = 1) { - const invalidCount = await GaqSummaryRepository.count({ where: { invalidatedAt: { [Op.not]: null } } }); - const remaining = Math.min(batchSize, invalidCount); - if (remaining === 0) { - return; - } - await dataSource.transaction(async () => { - for (let i = 0; i < remaining; i++) { - const invalidation = await GaqSummaryRepository.findOne({ - where: { invalidatedAt: { [Op.not]: null } }, - order: [['invalidatedAt', 'ASC']], - }); - if (!invalidation) { - return; - } - const { dataPassId, runNumber } = invalidation; - await this.calculateAndStoreGaqSummary(dataPassId, runNumber); - } + const rows = await GaqSummaryRepository.findAll({ + where: { invalidatedAt: { [Op.not]: null } }, + order: [['invalidatedAt', 'ASC']], + limit: batchSize, }); + + if (rows.length > 0) { + this._logger.infoMessage(`Processing ${rows.length} invalidated GAQ summaries`); + } + + await Promise.all(rows.map(({ dataPassId, runNumber }) => + this.calculateAndStoreGaqSummary(dataPassId, runNumber))); } } From 772a96c179e35f1f705a6a04e6f41a518c627856 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 5 May 2026 10:23:03 +0200 Subject: [PATCH 28/39] [O2B-1545] Add index on gaq_summaries.invalidated_at Create an index on invalidated_at column of gaq_summaries table to improve query performance, as it is commonly used to fetch the invalidated summary queue. --- .../v1/20260223120000-create-gaq-summary-tables.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js index 9366d050ce..639db07c5c 100644 --- a/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js +++ b/lib/database/migrations/v1/20260223120000-create-gaq-summary-tables.js @@ -58,6 +58,11 @@ module.exports = { defaultValue: Sequelize.literal('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'), }, }, { transaction }); + + await queryInterface.addIndex('gaq_summaries', { + name: 'gaq_summaries_invalidated_at_idx', + fields: ['invalidated_at'], + }, { transaction }); }), down: async (queryInterface) => queryInterface.sequelize.transaction(async (transaction) => { From 92e3c54af043f2eeb6a00c09dba982f05e8e4698 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 5 May 2026 12:32:20 +0200 Subject: [PATCH 29/39] [O2B-1563] Stealth fix: We do not need to soft delete flags that are already soft deleted The simplest way of invalidating GAQ summaries affected by the deletion of a dataPass' Flags is to just take the list of deleted flags. But this is inefficient if some flags were already deleted and thus had already had their summaries at the time. --- lib/server/services/qualityControlFlag/QcFlagService.js | 1 + test/api/qcFlags.test.js | 2 +- test/lib/server/services/gaq/GaqSummary.test.js | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index e3dd1a9dda..b1484df9b8 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -368,6 +368,7 @@ class QcFlagService { id: dataPassId, }, }, + where: { deleted: false }, raw: true, })).map(({ id, runNumber }) => ({ id, runNumber })); diff --git a/test/api/qcFlags.test.js b/test/api/qcFlags.test.js index ee67961439..092df4d883 100644 --- a/test/api/qcFlags.test.js +++ b/test/api/qcFlags.test.js @@ -1110,7 +1110,7 @@ module.exports = () => { .delete(`/api/qcFlags/perDataPass?dataPassId=${dataPassId}&token=${BkpRoles.DPG_ASYNC_QC_ADMIN}`); expect(response.status).to.be.equal(200); - expect(response.body.data.deletedCount).to.equal(11); // 9 from seeders, 2 created in POST requests previously in this test + expect(response.body.data.deletedCount).to.equal(10); // 9 from seeders, 2 created in POST requests previously in this test, 1 already soft-deleted }); }); diff --git a/test/lib/server/services/gaq/GaqSummary.test.js b/test/lib/server/services/gaq/GaqSummary.test.js index b4f6ae920d..62ad2a980e 100644 --- a/test/lib/server/services/gaq/GaqSummary.test.js +++ b/test/lib/server/services/gaq/GaqSummary.test.js @@ -89,7 +89,6 @@ module.exports = () => { await qcFlagService.deleteAllForDataPass(dataPassId); await expectInvalidation(dataPassId, 100); - await expectInvalidation(dataPassId, 105); }); it('should invalidate GAQ summary when GAQ detectors are explicitly set for a data pass and run', async () => { From 49430910ffa16cf397c9a9e912555ba65485b960 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 5 May 2026 12:46:12 +0200 Subject: [PATCH 30/39] [O2B-1564] Improve logging message for GAQWorker More details logged to help debugging the state of the GAQ worker. --- lib/server/services/gaq/GaqService.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/server/services/gaq/GaqService.js b/lib/server/services/gaq/GaqService.js index cee14a5c19..a2e33e6be9 100644 --- a/lib/server/services/gaq/GaqService.js +++ b/lib/server/services/gaq/GaqService.js @@ -170,14 +170,14 @@ class GaqService { * @return {Promise} promise */ async popNInvalidSummaryAndRecalculate(batchSize = 1) { - const rows = await GaqSummaryRepository.findAll({ + const { rows, count } = await GaqSummaryRepository.findAndCountAll({ where: { invalidatedAt: { [Op.not]: null } }, order: [['invalidatedAt', 'ASC']], limit: batchSize, }); if (rows.length > 0) { - this._logger.infoMessage(`Processing ${rows.length} invalidated GAQ summaries`); + this._logger.infoMessage(`Processing ${rows.length} out of ${count} invalidated GAQ summaries (batch size: ${batchSize})`); } await Promise.all(rows.map(({ dataPassId, runNumber }) => From 0586b3ef55e67f537cf99a67f8bb9b820d11225e Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Tue, 5 May 2026 22:44:28 +0200 Subject: [PATCH 31/39] [O2B-1564] Add warning message when the num of invalidated summaries consistently exceeds batch size Also move any GAQ worker related logging to within the GAQ worker itself (separation of concerns). --- lib/server/services/gaq/GaqService.js | 6 ++---- lib/server/services/gaq/GaqWorker.js | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/server/services/gaq/GaqService.js b/lib/server/services/gaq/GaqService.js index a2e33e6be9..74bb1360d7 100644 --- a/lib/server/services/gaq/GaqService.js +++ b/lib/server/services/gaq/GaqService.js @@ -176,12 +176,10 @@ class GaqService { limit: batchSize, }); - if (rows.length > 0) { - this._logger.infoMessage(`Processing ${rows.length} out of ${count} invalidated GAQ summaries (batch size: ${batchSize})`); - } - await Promise.all(rows.map(({ dataPassId, runNumber }) => this.calculateAndStoreGaqSummary(dataPassId, runNumber))); + + return { processedCount: rows.length, totalInvalidCount: count }; } } diff --git a/lib/server/services/gaq/GaqWorker.js b/lib/server/services/gaq/GaqWorker.js index a89671aba6..2393adb258 100644 --- a/lib/server/services/gaq/GaqWorker.js +++ b/lib/server/services/gaq/GaqWorker.js @@ -12,6 +12,8 @@ class GaqWorker { this._logger = LogManager.getLogger(GaqWorker.name); this._isPaused = false; this._isSynchronizing = false; + + this.batchSmallerThanInvalidCountWarningTimesOccurring = 0; } /** @@ -41,7 +43,23 @@ class GaqWorker { } this._isSynchronizing = true; try { - await gaqService.popNInvalidSummaryAndRecalculate(batchSize); + const { processedCount, totalInvalidCount } = await gaqService.popNInvalidSummaryAndRecalculate(batchSize); + + if (processedCount > 0) { + this._logger.infoMessage(`Processed ${processedCount} out of ${totalInvalidCount} ` + + `invalidated GAQ summaries (batch size: ${batchSize})`); + } + + if (totalInvalidCount > batchSize) { + this.batchSmallerThanInvalidCountWarningTimesOccurring += 1; + + if (this.batchSmallerThanInvalidCountWarningTimesOccurring >= 5) { + this._logger.warnMessage('For 5 iterations, there have been more invalidated GAQ summaries than the batch size ' + + `(${batchSize}). Consider increasing the batch size.`); + } + } else { + this.batchSmallerThanInvalidCountWarningTimesOccurring = 0; + } } catch (error) { this._logger.errorMessage(`Error recalculating GAQ summaries: ${error.message}\n${error.stack}`); } finally { From f5f686c906213b99e563f1ceb81882c53bb2d345 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Thu, 18 Jun 2026 12:15:32 +0200 Subject: [PATCH 32/39] [O2B-1545] Add a composite primary key on GAQSummary table Sequelize model omitted the primaryKey that migration file defines. --- lib/database/models/gaqSummary.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/database/models/gaqSummary.js b/lib/database/models/gaqSummary.js index efa76e04fd..3a12f62f33 100644 --- a/lib/database/models/gaqSummary.js +++ b/lib/database/models/gaqSummary.js @@ -17,9 +17,11 @@ module.exports = (sequelize) => { const GaqSummary = sequelize.define('GaqSummary', { dataPassId: { type: Sequelize.INTEGER, + primaryKey: true, }, runNumber: { type: Sequelize.INTEGER, + primaryKey: true, }, badRunCoverage: { type: Sequelize.FLOAT, From 9c1116b0944266fac0fe6b769d7ff4bce5230105 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Thu, 18 Jun 2026 13:18:25 +0200 Subject: [PATCH 33/39] [O2B-1563] Centralise GAQ summary invalidation and skip redundant work Extract invalidation into helpers. Guard GAQ updateRun reload so don't run reload unnecessarily. Add to notComputable field in model allowNull default params. Add more test coverage for invalidation on run patch. --- lib/database/models/gaqSummary.js | 2 + .../repositories/GaqSummaryRepository.js | 22 +++++++++++ .../services/gaq/GaqDetectorsService.js | 14 +------ .../qualityControlFlag/QcFlagService.js | 36 ++++++------------ lib/server/services/run/updateRun.js | 37 ++++++++++++------- .../server/services/gaq/GaqSummary.test.js | 20 ++++++++++ 6 files changed, 81 insertions(+), 50 deletions(-) diff --git a/lib/database/models/gaqSummary.js b/lib/database/models/gaqSummary.js index 3a12f62f33..ca2a1f69b8 100644 --- a/lib/database/models/gaqSummary.js +++ b/lib/database/models/gaqSummary.js @@ -40,6 +40,8 @@ module.exports = (sequelize) => { }, notComputable: { type: Sequelize.BOOLEAN, + allowNull: false, + defaultValue: false, }, invalidatedAt: { type: Sequelize.DATE(3), diff --git a/lib/database/repositories/GaqSummaryRepository.js b/lib/database/repositories/GaqSummaryRepository.js index 98986de513..81463d31c3 100644 --- a/lib/database/repositories/GaqSummaryRepository.js +++ b/lib/database/repositories/GaqSummaryRepository.js @@ -28,6 +28,28 @@ class GaqSummaryRepository extends Repository { constructor() { super(GaqSummary); } + + /** + * Mark the summary for a given (dataPassId, runNumber) as invalidated, creating the row if it does not yet exist + * + * @param {number} dataPassId data pass id + * @param {number} runNumber run number + * @return {Promise} resolves once the summary is invalidated + */ + async invalidate(dataPassId, runNumber) { + await this.upsert({ dataPassId, runNumber, invalidatedAt: new Date() }); + } + + /** + * Mark a list of summaries as invalidated in parallel + * + * @param {{ dataPassId: number, runNumber: number }[]} pairs the (dataPassId, runNumber) pairs to invalidate + * @return {Promise} resolves once all summaries are invalidated + */ + async invalidateMany(pairs) { + const invalidatedAt = new Date(); + await Promise.all(pairs.map(({ dataPassId, runNumber }) => this.upsert({ dataPassId, runNumber, invalidatedAt }))); + } } module.exports = new GaqSummaryRepository(); diff --git a/lib/server/services/gaq/GaqDetectorsService.js b/lib/server/services/gaq/GaqDetectorsService.js index 4411acdefa..4f6d466503 100644 --- a/lib/server/services/gaq/GaqDetectorsService.js +++ b/lib/server/services/gaq/GaqDetectorsService.js @@ -58,12 +58,7 @@ class GaqDetectorService { .map((detectorId) => ({ dataPassId, runNumber, detectorId }))); const createdEntries = await GaqDetectorRepository.insertAll(gaqEntries); - // Invalidate GAQ summaries for all affected runs - await Promise.all(runNumbers.map((runNumber) => GaqSummaryRepository.upsert({ - dataPassId, - runNumber, - invalidatedAt: new Date(), - }))); + await GaqSummaryRepository.invalidateMany(runNumbers.map((runNumber) => ({ dataPassId, runNumber }))); return createdEntries.map(gaqDetectorAdapter.toEntity); }); @@ -110,12 +105,7 @@ class GaqDetectorService { .map(({ id: detectorId }) => ({ dataPassId, runNumber, detectorId }))); const createdEntries = await GaqDetectorRepository.insertAll(gaqEntries); - // Invalidate GAQ summaries for all affected runs - await Promise.all(runNumbers.map((runNumber) => GaqSummaryRepository.upsert({ - dataPassId, - runNumber, - invalidatedAt: new Date(), - }))); + await GaqSummaryRepository.invalidateMany(runNumbers.map((runNumber) => ({ dataPassId, runNumber }))); return createdEntries.map(gaqDetectorAdapter.toEntity); }, { transaction }); diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index b1484df9b8..b6a647290a 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -210,13 +210,12 @@ class QcFlagService { ], }); - if (dataPass) { - // Invalidate GAQ summary for the dataPass and runNumber of the created flag - await GaqSummaryRepository.upsert({ - dataPassId: dataPass.id, - runNumber, - invalidatedAt: new Date(), - }); + /** + * Invalidate GAQ summary for the dataPass and runNumber of the created flag. + * Skip when `verify` is true: verifyFlag() above already invalidated the same (dataPassId, runNumber). + */ + if (dataPass && !verify) { + await GaqSummaryRepository.invalidate(dataPass.id, runNumber); } createdFlags.push(qcFlagAdapter.toEntity(createdFlag)); @@ -296,12 +295,8 @@ class QcFlagService { const { id, from, to, origin, createdById, runNumber, dplDetectorId, flagTypeId, createdAt } = qcFlag; if (dataPassId) { - // Invalidate GAQ summary for the dataPass and runNumber of the created flag - await GaqSummaryRepository.upsert({ - dataPassId: dataPassId, - runNumber, - invalidatedAt: new Date(), - }); + // Invalidate GAQ summary for the dataPass and runNumber of the deleted flag + await GaqSummaryRepository.invalidate(dataPassId, runNumber); } const qcFlagPropertiesToLog = { @@ -373,8 +368,7 @@ class QcFlagService { })).map(({ id, runNumber }) => ({ id, runNumber })); const qcFlagIds = qcFlagIdsToRunNumbers.map(({ id }) => id); - const runNumbers = new Set(); - qcFlagIdsToRunNumbers.map(({ runNumber }) => runNumbers.add(runNumber)); + const runNumbers = new Set(qcFlagIdsToRunNumbers.map(({ runNumber }) => runNumber)); await QcFlagRepository.updateAll( { deleted: true }, @@ -382,11 +376,7 @@ class QcFlagService { ); // Invalidate GAQ summary for the dataPass and all runNumbers of the deleted flags - await Promise.all(Array.from(runNumbers).map((runNumber) => GaqSummaryRepository.upsert({ - dataPassId, - runNumber, - invalidatedAt: new Date(), - }))); + await GaqSummaryRepository.invalidateMany(Array.from(runNumbers, (runNumber) => ({ dataPassId, runNumber }))); return qcFlagIds.length; }); @@ -429,11 +419,7 @@ class QcFlagService { // Invalidate GAQ summary if it's the first verification if (dataPassId && (!qcFlag.verifications || qcFlag.verifications.length === 0)) { - await GaqSummaryRepository.upsert({ - dataPassId, - runNumber: updatedQcFlag.runNumber, - invalidatedAt: new Date(), - }); + await GaqSummaryRepository.invalidate(dataPassId, updatedQcFlag.runNumber); } return updatedQcFlag; diff --git a/lib/server/services/run/updateRun.js b/lib/server/services/run/updateRun.js index a7833ed107..f4b9f0e69a 100644 --- a/lib/server/services/run/updateRun.js +++ b/lib/server/services/run/updateRun.js @@ -37,6 +37,20 @@ const { logSpecificRunTag } = require('./logEntriesCreation/logSpecificRunTag.js */ const TAGS_TO_LOG = ['Not for physics']; +/** + * Run patch fields whose values feed the qc_time_start / qc_time_end virtual columns + * + * @type {string[]} + */ +const QC_TIME_SOURCE_FIELDS = [ + 'firstTfTimestamp', + 'lastTfTimestamp', + 'timeTrgStart', + 'timeTrgEnd', + 'timeO2Start', + 'timeO2End', +]; + /** * Update the given run * @@ -211,20 +225,17 @@ exports.updateRun = async (identifier, payload, transaction) => { } } - // Need to reload the runModel as qcTimeStart and qcTimeEnd are virtual columns not in the patch so will stay stale after update - await runModel.reload(); - if (previousRun.qcTimeStart?.getTime() !== runModel.qcTimeStart?.getTime() - || previousRun.qcTimeEnd?.getTime() !== runModel.qcTimeEnd?.getTime()) { - const dataPassRuns = await DataPassRunRepository.findAll({ - attributes: ['dataPassId'], - where: { runNumber: runModel.runNumber }, - }); - for (const { dataPassId } of dataPassRuns) { - await GaqSummaryRepository.upsert({ - dataPassId, - runNumber: runModel.runNumber, - invalidatedAt: new Date(), + // Only check for invalidation when the patch could have changed them + if (QC_TIME_SOURCE_FIELDS.some((field) => field in runPatch)) { + // Reload because qcTimeStart/qcTimeEnd are virtual columns not in the patch and stay stale after update + await runModel.reload(); + if (previousRun.qcTimeStart?.getTime() !== runModel.qcTimeStart?.getTime() + || previousRun.qcTimeEnd?.getTime() !== runModel.qcTimeEnd?.getTime()) { + const dataPassRuns = await DataPassRunRepository.findAll({ + attributes: ['dataPassId'], + where: { runNumber: runModel.runNumber }, }); + await GaqSummaryRepository.invalidateMany(dataPassRuns.map(({ dataPassId }) => ({ dataPassId, runNumber: runModel.runNumber }))); } } diff --git a/test/lib/server/services/gaq/GaqSummary.test.js b/test/lib/server/services/gaq/GaqSummary.test.js index 62ad2a980e..c681b62b93 100644 --- a/test/lib/server/services/gaq/GaqSummary.test.js +++ b/test/lib/server/services/gaq/GaqSummary.test.js @@ -116,5 +116,25 @@ module.exports = () => { await expectInvalidation(dataPassId, runNumber); }); + + it('should NOT invalidate GAQ summary when updateRun patch does not touch QC time source fields', async () => { + await updateRun( + { runNumber }, + { runPatch: { definition: 'PHYSICS' } }, + ); + + await expectInvalidation(dataPassId, runNumber, true); + }); + + it('should NOT invalidate GAQ summary when a QC time source field is patched but qcTimeStart/qcTimeEnd do not change', async () => { + // Run 56 has a non-null time_trg_start, so qc_time_start resolves to time_trg_start regardless of time_o2_start. + // Patching only time_o2_start moves a source field but does not change the virtual qcTimeStart / qcTimeEnd, so no invalidation should fire. + await updateRun( + { runNumber }, + { runPatch: { timeO2Start: new Date('2019-08-08 18:00:00') } }, + ); + + await expectInvalidation(dataPassId, runNumber, true); + }); }); }; From 8992231f5214480046c01875d9765f23277f3617 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Fri, 19 Jun 2026 13:51:34 +0200 Subject: [PATCH 34/39] [O2B-1564] Overwrite summary values if couldn't be calculated --- lib/server/services/gaq/GaqService.js | 12 ++++++- .../server/services/gaq/GaqService.test.js | 31 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/lib/server/services/gaq/GaqService.js b/lib/server/services/gaq/GaqService.js index 74bb1360d7..6ab62d9673 100644 --- a/lib/server/services/gaq/GaqService.js +++ b/lib/server/services/gaq/GaqService.js @@ -161,7 +161,17 @@ class GaqService { */ async calculateAndStoreGaqSummary(dataPassId, runNumber) { const summary = await this._computeSummary(dataPassId, runNumber); - await GaqSummaryRepository.upsert({ dataPassId, runNumber, ...summary, notComputable: summary === null, invalidatedAt: null }); + await GaqSummaryRepository.upsert({ + dataPassId, + runNumber, + badRunCoverage: summary?.badRunCoverage ?? null, + explicitlyNotBadRunCoverage: summary?.explicitlyNotBadRunCoverage ?? null, + mcReproducibleCoverage: summary?.mcReproducibleCoverage ?? null, + missingVerificationsCount: summary?.missingVerificationsCount ?? null, + undefinedQualityPeriodsCount: summary?.undefinedQualityPeriodsCount ?? null, + notComputable: summary === null, + invalidatedAt: null, + }); } /** diff --git a/test/lib/server/services/gaq/GaqService.test.js b/test/lib/server/services/gaq/GaqService.test.js index baf6cbfb0b..709f0e13ee 100644 --- a/test/lib/server/services/gaq/GaqService.test.js +++ b/test/lib/server/services/gaq/GaqService.test.js @@ -80,6 +80,37 @@ module.exports = () => { const summary = await findSummary(dataPassId, 49); expect(summary).to.not.be.null; expect(summary.notComputable).to.be.true; + + await GaqSummaryRepository.removeAll({ where: { dataPassId, runNumber: 49 } }); + }); + + it('should clear stale coverage fields when a previously-computable row becomes notComputable', async () => { + // Seed a row that has values but will become notComputable after recalculation due to missing QC flags + const staleRunNumber = 49; + await GaqSummaryRepository.upsert({ + dataPassId, + runNumber: staleRunNumber, + badRunCoverage: 0.5, + explicitlyNotBadRunCoverage: 0.4, + mcReproducibleCoverage: 0.1, + missingVerificationsCount: 2, + undefinedQualityPeriodsCount: 1, + notComputable: false, + }); + + // Run 49 has no QC flags seeded for data pass 1, so _computeSummary returns null + await gaqService.calculateAndStoreGaqSummary(dataPassId, staleRunNumber); + + const summary = await findSummary(dataPassId, staleRunNumber); + expect(summary).to.not.be.null; + expect(summary.notComputable).to.be.true; + expect(summary.badRunCoverage).to.be.null; + expect(summary.explicitlyNotBadRunCoverage).to.be.null; + expect(summary.mcReproducibleCoverage).to.be.null; + expect(summary.missingVerificationsCount).to.be.null; + expect(summary.undefinedQualityPeriodsCount).to.be.null; + + await GaqSummaryRepository.removeAll({ where: { dataPassId, runNumber: staleRunNumber } }); }); }); From 7481ec1bf9b9bdf57cd6ff5227101a0fe60da1af Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Fri, 19 Jun 2026 14:50:25 +0200 Subject: [PATCH 35/39] [O2B-1564] Protect invalidatedAt from race condition Protects invalidatedAt from being set to null if the datapass and runNumber becomes invalid for whatever reason in the middle of computeSummary. ComputeSummary would overwrite the invalidatedAt column with null at the end. Now it performs a check to prevent this. --- lib/server/services/gaq/GaqService.js | 34 ++++++++++++++---- .../server/services/gaq/GaqService.test.js | 35 +++++++++++++++++++ 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/lib/server/services/gaq/GaqService.js b/lib/server/services/gaq/GaqService.js index 6ab62d9673..a2b145ce3b 100644 --- a/lib/server/services/gaq/GaqService.js +++ b/lib/server/services/gaq/GaqService.js @@ -157,11 +157,14 @@ class GaqService { * Calculate and store GAQ summary for given data pass and run * @param {number} dataPassId id of data pass * @param {number} runNumber run number + * @param {object} [options] additional options + * @param {Date} [options.expectedInvalidatedAt] if provided, invalidatedAt will only be cleared if it is equal to the provided value * @return {Promise} promise */ - async calculateAndStoreGaqSummary(dataPassId, runNumber) { + async calculateAndStoreGaqSummary(dataPassId, runNumber, { expectedInvalidatedAt } = {}) { const summary = await this._computeSummary(dataPassId, runNumber); - await GaqSummaryRepository.upsert({ + + const fields = { dataPassId, runNumber, badRunCoverage: summary?.badRunCoverage ?? null, @@ -170,8 +173,27 @@ class GaqService { missingVerificationsCount: summary?.missingVerificationsCount ?? null, undefinedQualityPeriodsCount: summary?.undefinedQualityPeriodsCount ?? null, notComputable: summary === null, - invalidatedAt: null, - }); + }; + + if (expectedInvalidatedAt === undefined) { + // No expected invalidation time provided, just upsert the summary + await GaqSummaryRepository.upsert({ dataPassId, runNumber, ...fields, invalidatedAt: null }); + return; + }; + + // Only clear invalidatedAt if it hasn't been changed during compute + const [rows] = await GaqSummaryRepository.updateAll( + { ...fields, invalidatedAt: null }, + { where: { dataPassId, runNumber, invalidatedAt: expectedInvalidatedAt } }, + ); + + if (rows === 0) { + // Write fresh summary fields but leave invalidatedAt unchanged + await GaqSummaryRepository.updateAll( + { ...fields }, + { where: { dataPassId, runNumber } }, + ); + } } /** @@ -186,8 +208,8 @@ class GaqService { limit: batchSize, }); - await Promise.all(rows.map(({ dataPassId, runNumber }) => - this.calculateAndStoreGaqSummary(dataPassId, runNumber))); + await Promise.all(rows.map(({ dataPassId, runNumber, invalidatedAt }) => + this.calculateAndStoreGaqSummary(dataPassId, runNumber, { expectedInvalidatedAt: invalidatedAt }))); return { processedCount: rows.length, totalInvalidCount: count }; } diff --git a/test/lib/server/services/gaq/GaqService.test.js b/test/lib/server/services/gaq/GaqService.test.js index 709f0e13ee..b2a96c49e9 100644 --- a/test/lib/server/services/gaq/GaqService.test.js +++ b/test/lib/server/services/gaq/GaqService.test.js @@ -12,6 +12,7 @@ */ const { expect } = require('chai'); +const sinon = require('sinon'); const { resetDatabaseContent } = require('../../../../utilities/resetDatabaseContent.js'); const { repositories: { GaqSummaryRepository} } = require('../../../../../lib/database'); const { gaqService } = require('../../../../../lib/server/services/gaq/GaqService.js'); @@ -112,6 +113,40 @@ module.exports = () => { await GaqSummaryRepository.removeAll({ where: { dataPassId, runNumber: staleRunNumber } }); }); + + it('should not clear invalidatedAt if the row was re-invalidated during compute', async () => { + // Seed an initial invalidation so the row has a known invalidatedAt + await GaqSummaryRepository.invalidate(dataPassId, runNumber); + const initial = await findSummary(dataPassId, runNumber); + expect(initial.invalidatedAt).to.not.be.null; + + // Simulate a concurrent invalidate by mocking _computeSummary to invalidate the row again but still return a valid summary + sinon.stub(gaqService, '_computeSummary').callsFake(async () => { + await GaqSummaryRepository.invalidate(dataPassId, runNumber); + return { + badRunCoverage: 0, + explicitlyNotBadRunCoverage: 1, + mcReproducibleCoverage: 0, + missingVerificationsCount: 0, + undefinedQualityPeriodsCount: 0, + }; + }); + + try { + await gaqService.calculateAndStoreGaqSummary( + dataPassId, + runNumber, + { expectedInvalidatedAt: initial.invalidatedAt }, + ); + } finally { + sinon.restore(); + } + + const after = await findSummary(dataPassId, runNumber); + // InvalidatedAt should not have been cleared because the row was re-invalidated during compute + expect(after.invalidatedAt).to.not.be.null; + expect(after.invalidatedAt).to.be.greaterThan(initial.invalidatedAt); + }); }); describe('popNInvalidSummaryAndRecalculate', () => { From fe8062c531e82d74b583148bcab4bb35e7652a07 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Fri, 19 Jun 2026 15:15:33 +0200 Subject: [PATCH 36/39] [O2B-1564] Fixed silent error in test --- test/lib/server/services/gaq/GaqSummary.test.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/lib/server/services/gaq/GaqSummary.test.js b/test/lib/server/services/gaq/GaqSummary.test.js index 5bcde3853e..21d10b7fa7 100644 --- a/test/lib/server/services/gaq/GaqSummary.test.js +++ b/test/lib/server/services/gaq/GaqSummary.test.js @@ -242,14 +242,13 @@ module.exports = () => { // First call — will be held open by the slow stub const firstCall = gaqWorker.recalculateGaqSummaries(1); - // Second call — should be skipped because _isSynchronizing is true await gaqWorker.recalculateGaqSummaries(1); // Stub should only have been called once expect(stub.callCount).to.equal(1); - // Release the first call - resolveFirst(); + // Release the first call with the shape popNInvalidSummaryAndRecalculate normally returns + resolveFirst({ processedCount: 0, totalInvalidCount: 0 }); await firstCall; } finally { sinon.restore(); From 9851455b67181978d35b6d2802151386707477c1 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Fri, 19 Jun 2026 15:16:58 +0200 Subject: [PATCH 37/39] [O2B-1564] Modify pause behaviour to account for in-flight recalculation --- lib/server/services/gaq/GaqWorker.js | 35 ++++++++++++++----- .../server/services/gaq/GaqSummary.test.js | 1 + test/utilities/resetDatabaseContent.js | 6 ++-- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/lib/server/services/gaq/GaqWorker.js b/lib/server/services/gaq/GaqWorker.js index 2393adb258..0a5cfb466d 100644 --- a/lib/server/services/gaq/GaqWorker.js +++ b/lib/server/services/gaq/GaqWorker.js @@ -11,17 +11,25 @@ class GaqWorker { constructor() { this._logger = LogManager.getLogger(GaqWorker.name); this._isPaused = false; - this._isSynchronizing = false; + this._currentRun = null; this.batchSmallerThanInvalidCountWarningTimesOccurring = 0; } /** - * Pause the worker so it skips processing on the next scheduled calls - * @return {void} + * Pause the worker so it skips future scheduled calls, and await any in-flight call to finish + * so callers can safely mutate shared state (e.g. drop tables in tests) once this resolves + * @return {Promise} resolves once the worker is idle and paused */ - pause() { + async pause() { this._isPaused = true; + if (this._currentRun) { + try { + await this._currentRun; + } catch { + // Already logged inside _doRecalculate + } + } } /** @@ -38,10 +46,23 @@ class GaqWorker { * @return {Promise} promise */ async recalculateGaqSummaries(batchSize) { - if (this._isSynchronizing || this._isPaused) { + if (this._isPaused || this._currentRun) { return; } - this._isSynchronizing = true; + this._currentRun = this._doRecalculate(batchSize); + try { + await this._currentRun; + } finally { + this._currentRun = null; + } + } + + /** + * Run a single recalculation pass; errors are logged but not rethrown + * @param {number} batchSize number of invalid summaries to recalculate + * @return {Promise} promise + */ + async _doRecalculate(batchSize) { try { const { processedCount, totalInvalidCount } = await gaqService.popNInvalidSummaryAndRecalculate(batchSize); @@ -62,8 +83,6 @@ class GaqWorker { } } catch (error) { this._logger.errorMessage(`Error recalculating GAQ summaries: ${error.message}\n${error.stack}`); - } finally { - this._isSynchronizing = false; } } } diff --git a/test/lib/server/services/gaq/GaqSummary.test.js b/test/lib/server/services/gaq/GaqSummary.test.js index 21d10b7fa7..f613dc4d5c 100644 --- a/test/lib/server/services/gaq/GaqSummary.test.js +++ b/test/lib/server/services/gaq/GaqSummary.test.js @@ -242,6 +242,7 @@ module.exports = () => { // First call — will be held open by the slow stub const firstCall = gaqWorker.recalculateGaqSummaries(1); + // Second call — should be skipped because a previous run is still in flight await gaqWorker.recalculateGaqSummaries(1); // Stub should only have been called once diff --git a/test/utilities/resetDatabaseContent.js b/test/utilities/resetDatabaseContent.js index 704a986219..79d9996f7d 100644 --- a/test/utilities/resetDatabaseContent.js +++ b/test/utilities/resetDatabaseContent.js @@ -15,9 +15,9 @@ const { database } = require('../../lib/application.js'); const { gaqWorker } = require('../../lib/server/services/gaq/GaqWorker.js'); exports.resetDatabaseContent = async () => { - // Pause GAQ worker when resetDatabaseContent() runs in between tests in the different test suites - // Otherwise, worker fails as the invalidation table is DROPPED. This avoids an ERROR message appearing in test logs even if the suite passed - gaqWorker.pause(); + // Pause GAQ worker and await any in-flight call before dropping tables, otherwise a tick + // already past the guard would hit dropped tables and log a spurious ERROR + await gaqWorker.pause(); await database.dropAllTables(); await database.migrate(); await database.seed(); From d7dc453b6e7f3ed7f90d71b249c8a61c17bcaaf4 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Fri, 19 Jun 2026 15:27:25 +0200 Subject: [PATCH 38/39] [O2B-1564] Remove extra where option --- lib/server/services/qualityControlFlag/QcFlagService.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 70b4f92194..e389375453 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -364,7 +364,6 @@ class QcFlagService { id: dataPassId, }, }, - where: { deleted: false }, raw: true, })).map(({ id, runNumber }) => ({ id, runNumber })); From 35272c745f167fe9e80956984fc57722c50c0059 Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Fri, 19 Jun 2026 17:04:23 +0200 Subject: [PATCH 39/39] [O2B-1564] Make batchSize dynamic between a min and max Add a helper that logs a warning when the env var is set to an invalid value instead of silently falling back to the default. If batchSize is not enough log the first warning after 5 consecutive overflow ticks, then every 30 ticks while it persists and recovery line when the streak ends so operators can grep for "backlog recovered". Log at debug level when the worker is paused or resumed. --- docker-compose.dev.yml | 3 +- docker-compose.test.yml | 3 +- lib/application.js | 2 +- lib/config/services.js | 24 +++++++- lib/server/services/gaq/GaqWorker.js | 59 +++++++++++++++---- .../server/services/gaq/GaqSummary.test.js | 8 +-- 6 files changed, 78 insertions(+), 21 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index f8b026a81a..c188aa5b70 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -12,7 +12,8 @@ services: GRPC_AUTHENTICATED_ORIGIN: '[::]:4002' GAQ_ENABLE_RECALCULATION: "True" GAQ_RECALCULATION_PERIOD: 10000 - GAQ_RECALCULATION_BATCH_SIZE: 20 + GAQ_RECALCULATION_MIN_BATCH_SIZE: 5 + GAQ_RECALCULATION_MAX_BATCH_SIZE: 50 ports: - "4000:4000" - "4001:4001" diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 3d4bce9081..a505b1022e 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -12,7 +12,8 @@ services: CCDB_SYNCHRONIZATION_PERIOD: 3153600000000 # 100y in milliseconds, to be sure all the runs are included when testing sync GAQ_ENABLE_RECALCULATION: "True" GAQ_RECALCULATION_PERIOD: 1000 - GAQ_RECALCULATION_BATCH_SIZE: 1 + GAQ_RECALCULATION_MIN_BATCH_SIZE: 1 + GAQ_RECALCULATION_MAX_BATCH_SIZE: 1 restart: "no" database: diff --git a/lib/application.js b/lib/application.js index f7d64dce01..ff39c2ed25 100644 --- a/lib/application.js +++ b/lib/application.js @@ -135,7 +135,7 @@ class BookkeepingApplication { if (gaqConfig.enableRecalculation) { this.scheduledProcessesManager.schedule( - () => gaqWorker.recalculateGaqSummaries(gaqConfig.batchSize), + () => gaqWorker.recalculateGaqSummaries(gaqConfig.minBatchSize, gaqConfig.maxBatchSize), { wait: 10 * 1000, every: gaqConfig.recalculationPeriod, diff --git a/lib/config/services.js b/lib/config/services.js index e2a548565b..599cc1ad13 100644 --- a/lib/config/services.js +++ b/lib/config/services.js @@ -27,6 +27,25 @@ const { CCDB_RUN_INFO_URL, } = process.env ?? {}; +/** + * Parse a positive integer env var, falling back to the default and warning if the value is invalid + * + * @param {string|undefined} raw the raw env var value + * @param {number} defaultValue value to use when raw is unset or invalid + * @param {string} name env var name (for the warning message) + * @return {number} the parsed value or the default + */ +const parsePositiveInt = (raw, defaultValue, name) => { + if (raw === undefined) return defaultValue; + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed <= 0) { + // eslint-disable-next-line no-console + console.warn(`Invalid ${name}=${JSON.stringify(raw)}; falling back to ${defaultValue}`); + return defaultValue; + } + return parsed; +}; + exports.services = { enableHousekeeping: process.env?.ENABLE_HOUSEKEEPING?.toLowerCase() === 'true', userCertificate: { @@ -71,7 +90,8 @@ exports.services = { gaq: { enableRecalculation: process.env?.GAQ_ENABLE_RECALCULATION?.toLowerCase() === 'true', - recalculationPeriod: Number(process.env?.GAQ_RECALCULATION_PERIOD) || 30 * 1000, // 30s in milliseconds - batchSize: Number(process.env?.GAQ_RECALCULATION_BATCH_SIZE) || 10, + recalculationPeriod: parsePositiveInt(process.env?.GAQ_RECALCULATION_PERIOD, 30 * 1000, 'GAQ_RECALCULATION_PERIOD'), // 30s default + minBatchSize: parsePositiveInt(process.env?.GAQ_RECALCULATION_MIN_BATCH_SIZE, 1, 'GAQ_RECALCULATION_MIN_BATCH_SIZE'), + maxBatchSize: parsePositiveInt(process.env?.GAQ_RECALCULATION_MAX_BATCH_SIZE, 100, 'GAQ_RECALCULATION_MAX_BATCH_SIZE'), }, }; diff --git a/lib/server/services/gaq/GaqWorker.js b/lib/server/services/gaq/GaqWorker.js index 0a5cfb466d..5e4cbccf7c 100644 --- a/lib/server/services/gaq/GaqWorker.js +++ b/lib/server/services/gaq/GaqWorker.js @@ -1,6 +1,12 @@ const { gaqService } = require('../../services/gaq/GaqService.js'); const { LogManager } = require('@aliceo2/web-ui'); +// Tolerate brief blips before warning so transient overshoots don't spam logs +const OVERFLOW_THRESHOLD_TICKS = 5; + +// While the overflow persists, re-warn at this cadence so operators see "still bad" without log floods +const OVERFLOW_REMINDER_EVERY_TICKS = 30; + /** * Worker responsible for processing pending GAQ summary invalidations */ @@ -13,7 +19,10 @@ class GaqWorker { this._isPaused = false; this._currentRun = null; - this.batchSmallerThanInvalidCountWarningTimesOccurring = 0; + // Adaptive batch size for the next tick. Null on first tick → falls back to the passed-in min. + this._nextBatchSize = null; + + this._overflowConsecutiveTicks = 0; } /** @@ -22,6 +31,9 @@ class GaqWorker { * @return {Promise} resolves once the worker is idle and paused */ async pause() { + if (!this._isPaused) { + this._logger.infoMessage('Worker paused'); + } this._isPaused = true; if (this._currentRun) { try { @@ -37,19 +49,24 @@ class GaqWorker { * @return {void} */ resume() { + if (this._isPaused) { + this._logger.infoMessage('Worker resumed'); + } this._isPaused = false; } /** * Process pending GAQ summary invalidations. Skips if a previous call is still in progress or if paused. - * @param {number} batchSize number of invalid summaries to recalculate + * The batch size for this tick is clamped between min and max and adapts to the observed backlog. + * @param {number} minBatchSize lower bound on rows to fetch per tick + * @param {number} maxBatchSize upper bound on rows to fetch per tick * @return {Promise} promise */ - async recalculateGaqSummaries(batchSize) { + async recalculateGaqSummaries(minBatchSize, maxBatchSize) { if (this._isPaused || this._currentRun) { return; } - this._currentRun = this._doRecalculate(batchSize); + this._currentRun = this._doRecalculate(minBatchSize, maxBatchSize); try { await this._currentRun; } finally { @@ -59,27 +76,45 @@ class GaqWorker { /** * Run a single recalculation pass; errors are logged but not rethrown - * @param {number} batchSize number of invalid summaries to recalculate + * @param {number} minBatchSize lower bound on rows to fetch per tick + * @param {number} maxBatchSize upper bound on rows to fetch per tick * @return {Promise} promise */ - async _doRecalculate(batchSize) { + async _doRecalculate(minBatchSize, maxBatchSize) { + const clamp = (n) => Math.min(maxBatchSize, Math.max(minBatchSize, n)); + const batchSize = clamp(this._nextBatchSize ?? minBatchSize); + try { const { processedCount, totalInvalidCount } = await gaqService.popNInvalidSummaryAndRecalculate(batchSize); + // Adapt next tick's batch size to the observed backlog (clamped to the bounds) + this._nextBatchSize = clamp(totalInvalidCount); + if (processedCount > 0) { this._logger.infoMessage(`Processed ${processedCount} out of ${totalInvalidCount} ` + `invalidated GAQ summaries (batch size: ${batchSize})`); } - if (totalInvalidCount > batchSize) { - this.batchSmallerThanInvalidCountWarningTimesOccurring += 1; + // Overflow: backlog still exceeds the max even at the largest batch we'll fetch + if (totalInvalidCount > maxBatchSize) { + this._overflowConsecutiveTicks += 1; - if (this.batchSmallerThanInvalidCountWarningTimesOccurring >= 5) { - this._logger.warnMessage('For 5 iterations, there have been more invalidated GAQ summaries than the batch size ' - + `(${batchSize}). Consider increasing the batch size.`); + const ticks = this._overflowConsecutiveTicks; + const firstWarning = ticks === OVERFLOW_THRESHOLD_TICKS; + const reminderDue = ticks > OVERFLOW_THRESHOLD_TICKS + && (ticks - OVERFLOW_THRESHOLD_TICKS) % OVERFLOW_REMINDER_EVERY_TICKS === 0; + + if (firstWarning || reminderDue) { + this._logger.warnMessage(`Invalidated GAQ summary backlog (${totalInvalidCount}) has exceeded ` + + `the max batch size (${maxBatchSize}) for ${ticks} consecutive ticks. ` + + 'Consider raising GAQ_RECALCULATION_MAX_BATCH_SIZE or shortening GAQ_RECALCULATION_PERIOD.'); } } else { - this.batchSmallerThanInvalidCountWarningTimesOccurring = 0; + if (this._overflowConsecutiveTicks >= OVERFLOW_THRESHOLD_TICKS) { + this._logger.infoMessage(`GAQ summary backlog recovered after ${this._overflowConsecutiveTicks} ` + + `consecutive overflow ticks (current backlog: ${totalInvalidCount})`); + } + this._overflowConsecutiveTicks = 0; } } catch (error) { this._logger.errorMessage(`Error recalculating GAQ summaries: ${error.message}\n${error.stack}`); diff --git a/test/lib/server/services/gaq/GaqSummary.test.js b/test/lib/server/services/gaq/GaqSummary.test.js index f613dc4d5c..fc15274711 100644 --- a/test/lib/server/services/gaq/GaqSummary.test.js +++ b/test/lib/server/services/gaq/GaqSummary.test.js @@ -220,8 +220,8 @@ module.exports = () => { await expectInvalidation(1, 106); await expectInvalidation(1, 107); - // Manually call the worker with batchSize=2 to process both in one go - await gaqWorker.recalculateGaqSummaries(2); + // Manually call the worker with min/max batchSize=2 to process both in one go + await gaqWorker.recalculateGaqSummaries(2, 2); await expectInvalidation(1, 106, true); await expectInvalidation(1, 107, true); @@ -240,10 +240,10 @@ module.exports = () => { try { // First call — will be held open by the slow stub - const firstCall = gaqWorker.recalculateGaqSummaries(1); + const firstCall = gaqWorker.recalculateGaqSummaries(1, 1); // Second call — should be skipped because a previous run is still in flight - await gaqWorker.recalculateGaqSummaries(1); + await gaqWorker.recalculateGaqSummaries(1, 1); // Stub should only have been called once expect(stub.callCount).to.equal(1);