From f4a040325ed7308e10c66ea9797d916a483012e8 Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Mon, 30 Mar 2026 18:48:26 -0400 Subject: [PATCH 1/6] feat(datastore): add DialectHelper SQL abstraction + PostgreSQL dialect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces a DialectHelper interface that abstracts MySQL-specific SQL constructs (INSERT IGNORE, ON DUPLICATE KEY UPDATE, GROUP_CONCAT, JSON_EXTRACT, IFNULL, error classification, etc.) behind a common API. - dialect.go: DialectHelper interface definition - dialect_mysql.go: MySQL implementation (behaviour-preserving, existing tests pass) - dialect_postgres.go: PostgreSQL implementation via SQL rewriting - server/platform/postgres/: pgx driver, rebind SQL rewriter, PG error classification - server/goose/: dual-dialect migration support (MySQL and PostgreSQL) - server/config/: Driver field to select mysql (default) or postgres - go.mod: add jackc/pgx/v5 PostgreSQL driver dependency The rebind driver transparently rewrites MySQL SQL to PostgreSQL-compatible SQL at query time: IFNULL→COALESCE, DATE_ADD/DATE_SUB→INTERVAL, IF()→CASE, ? placeholders→$N, boolean casting, JSON path translation, etc. --- go.mod | 4 + go.sum | 8 + server/config/config.go | 6 + server/datastore/mysql/dialect.go | 101 ++ server/datastore/mysql/dialect_mysql.go | 102 ++ server/datastore/mysql/dialect_mysql_test.go | 67 + server/datastore/mysql/dialect_postgres.go | 175 ++ .../datastore/mysql/dialect_postgres_test.go | 115 ++ server/goose/dialect.go | 14 +- server/goose/migrate.go | 19 + server/goose/migrate_test.go | 71 +- server/goose/migration.go | 49 +- server/platform/postgres/common.go | 31 + server/platform/postgres/errors.go | 114 ++ server/platform/postgres/rebind_driver.go | 1590 +++++++++++++++++ 15 files changed, 2457 insertions(+), 9 deletions(-) create mode 100644 server/datastore/mysql/dialect.go create mode 100644 server/datastore/mysql/dialect_mysql.go create mode 100644 server/datastore/mysql/dialect_mysql_test.go create mode 100644 server/datastore/mysql/dialect_postgres.go create mode 100644 server/datastore/mysql/dialect_postgres_test.go create mode 100644 server/platform/postgres/common.go create mode 100644 server/platform/postgres/errors.go create mode 100644 server/platform/postgres/rebind_driver.go diff --git a/go.mod b/go.mod index 191b01568e5..38892028e60 100644 --- a/go.mod +++ b/go.mod @@ -286,6 +286,10 @@ require ( github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.15 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect diff --git a/go.sum b/go.sum index fd07c888dd5..d9bbfb160ba 100644 --- a/go.sum +++ b/go.sum @@ -543,6 +543,14 @@ github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+h github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= diff --git a/server/config/config.go b/server/config/config.go index 4127a21bd4f..50c134f0ed4 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -39,6 +39,9 @@ const ( // MysqlConfig defines configs related to MySQL type MysqlConfig struct { + // Driver selects the database driver. Only "mysql" is valid in Phase 1. + // Future values: "postgres" (Phase 4+). + Driver string `yaml:"driver"` Protocol string `yaml:"protocol"` Address string `yaml:"address"` Username string `yaml:"username"` @@ -1128,6 +1131,8 @@ func (t *TLS) ToTLSConfig() (*tls.Config, error) { // filled into the FleetConfig struct func (man Manager) addConfigs() { addMysqlConfig := func(prefix, defaultAddr, usageSuffix string) { + man.addConfigString(prefix+".driver", "", + "Database driver: mysql (default) or postgres"+usageSuffix) man.addConfigString(prefix+".protocol", "tcp", "MySQL server communication protocol (tcp,unix,...)"+usageSuffix) man.addConfigString(prefix+".address", defaultAddr, @@ -1637,6 +1642,7 @@ func (man Manager) LoadConfig() FleetConfig { loadMysqlConfig := func(prefix string) MysqlConfig { return MysqlConfig{ + Driver: man.getConfigString(prefix + ".driver"), Protocol: man.getConfigString(prefix + ".protocol"), Address: man.getConfigString(prefix + ".address"), Username: man.getConfigString(prefix + ".username"), diff --git a/server/datastore/mysql/dialect.go b/server/datastore/mysql/dialect.go new file mode 100644 index 00000000000..54ed0d2337d --- /dev/null +++ b/server/datastore/mysql/dialect.go @@ -0,0 +1,101 @@ +package mysql + +import "github.com/doug-martin/goqu/v9" + +// DialectHelper abstracts SQL dialect differences between MySQL and PostgreSQL. +// All runtime SQL that is MySQL-specific must go through this interface so that +// a PostgreSQL implementation can substitute equivalent syntax. +// +// Upsert methods are fragment-based: they return SQL fragments (prefix or suffix) +// that compose into any query shape — single-row, multi-row batch, INSERT...SELECT. +type DialectHelper interface { + // ---- Upsert fragments ---- + + // InsertIgnoreInto returns the INSERT prefix for ignoring duplicate-key errors. + // MySQL: "INSERT IGNORE INTO" + // PostgreSQL: "INSERT INTO" + // For PostgreSQL, the caller must also append OnConflictDoNothing() to the query. + InsertIgnoreInto() string + + // ReplaceInto returns the REPLACE INTO prefix (MySQL) or "INSERT INTO" (PostgreSQL). + // MySQL: "REPLACE INTO" + // PostgreSQL: "INSERT INTO" + // For PostgreSQL, the caller must also append OnDuplicateKey() with all non-key + // columns to achieve REPLACE semantics (upsert all columns). + ReplaceInto() string + + // OnDuplicateKey returns the upsert conflict-handling suffix. + // MySQL: "ON DUPLICATE KEY UPDATE " + updateClause + // PostgreSQL: "ON CONFLICT (" + conflictTarget + ") DO UPDATE SET " + translated + // The updateClause uses MySQL syntax (e.g., "name=VALUES(name), updated_at=NOW()"). + // The PostgreSQL implementation translates VALUES(col) → EXCLUDED.col. + OnDuplicateKey(conflictTarget, updateClause string) string + + // OnConflictDoNothing returns the suffix for suppressing duplicate-key errors. + // MySQL: "" (handled by InsertIgnoreInto prefix) + // PostgreSQL: " ON CONFLICT (" + conflictTarget + ") DO NOTHING" + OnConflictDoNothing(conflictTarget string) string + + // ---- Aggregate & expression functions ---- + + // GroupConcat returns a GROUP_CONCAT (MySQL) or STRING_AGG (PostgreSQL) + // expression aggregating expr with the given separator. + GroupConcat(expr, separator string) string + + // JSONAgg returns a JSON_ARRAYAGG (MySQL) or json_agg (PostgreSQL) expression. + JSONAgg(expr string) string + + // JSONExtract returns an expression that extracts a value from a JSON column + // at the given path. MySQL: JSON_EXTRACT(col, path), PG: col->'path'. + JSONExtract(col, path string) string + + // JSONUnquoteExtract returns an expression that extracts a scalar string from + // a JSON column. MySQL: col->>'path' / JSON_UNQUOTE(JSON_EXTRACT(...)), + // PostgreSQL: col->>'path'. + JSONUnquoteExtract(col, path string) string + + // JSONBuildObject returns an expression that constructs a JSON object from + // alternating key/value strings. MySQL: JSON_OBJECT(k,v,...), + // PostgreSQL: jsonb_build_object(k,v,...). + JSONBuildObject(keyVals ...string) string + + // FindInSet returns an expression equivalent to MySQL FIND_IN_SET(needle, col). + // PostgreSQL: needle = ANY(string_to_array(col, ',')). + FindInSet(needle, col string) string + + // FullTextMatch returns a full-text search predicate. + // MySQL: MATCH(cols...) AGAINST (query IN BOOLEAN MODE), + // PostgreSQL: to_tsvector('english', col) @@ plainto_tsquery('english', query). + FullTextMatch(cols []string, query string) string + + // RegexpMatch returns a regular-expression match predicate. + // MySQL: col REGEXP pattern, PostgreSQL: col ~ pattern. + RegexpMatch(col, pattern string) string + + // ---- Goqu ---- + + // GoquDialect returns the goqu dialect wrapper appropriate for this driver. + GoquDialect() goqu.DialectWrapper + + // ---- Error classification ---- + + // IsDuplicate returns true if err is a unique-constraint violation. + IsDuplicate(err error) bool + + // IsForeignKey returns true if err is a foreign-key constraint violation. + IsForeignKey(err error) bool + + // IsReadOnly returns true if err indicates the server is in read-only mode. + IsReadOnly(err error) bool + + // IsBadConnection returns true if err is a connection-level error that + // justifies retrying on a new connection. + IsBadConnection(err error) bool + + // ReturningID returns " RETURNING id" for PostgreSQL (to be appended to + // INSERT statements) or "" for MySQL (which uses LastInsertId instead). + ReturningID() string + + // IsPostgres returns true if the dialect is PostgreSQL. + IsPostgres() bool +} diff --git a/server/datastore/mysql/dialect_mysql.go b/server/datastore/mysql/dialect_mysql.go new file mode 100644 index 00000000000..c3d4b0081cf --- /dev/null +++ b/server/datastore/mysql/dialect_mysql.go @@ -0,0 +1,102 @@ +package mysql + +import ( + "fmt" + "strings" + + "github.com/doug-martin/goqu/v9" + _ "github.com/doug-martin/goqu/v9/dialect/mysql" // register mysql dialect + common_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql" +) + +// mysqlDialect implements DialectHelper for MySQL / MariaDB. +// Every method returns exactly the SQL currently inlined across the datastore +// implementation — this is a pure structural refactoring with no behaviour change. +type mysqlDialect struct{} + +// Compile-time assertion that mysqlDialect satisfies DialectHelper. +var _ DialectHelper = mysqlDialect{} + +// InsertIgnoreInto returns "INSERT IGNORE INTO". +func (mysqlDialect) InsertIgnoreInto() string { return "INSERT IGNORE INTO" } + +// ReplaceInto returns "REPLACE INTO". +func (mysqlDialect) ReplaceInto() string { return "REPLACE INTO" } + +// OnDuplicateKey returns: ON DUPLICATE KEY UPDATE +// The updateClause is passed through verbatim (MySQL-native syntax). +func (mysqlDialect) OnDuplicateKey(_, updateClause string) string { + return "ON DUPLICATE KEY UPDATE " + updateClause +} + +// OnConflictDoNothing returns "" — MySQL handles ignore via the INSERT IGNORE prefix. +func (mysqlDialect) OnConflictDoNothing(_ string) string { return "" } + +// GroupConcat builds: GROUP_CONCAT( SEPARATOR '') +func (mysqlDialect) GroupConcat(expr, separator string) string { + return fmt.Sprintf("GROUP_CONCAT(%s SEPARATOR '%s')", expr, separator) +} + +// JSONAgg builds: JSON_ARRAYAGG() +func (mysqlDialect) JSONAgg(expr string) string { + return fmt.Sprintf("JSON_ARRAYAGG(%s)", expr) +} + +// JSONExtract builds: JSON_EXTRACT(, '') +func (mysqlDialect) JSONExtract(col, path string) string { + return fmt.Sprintf("JSON_EXTRACT(%s, '%s')", col, path) +} + +// JSONUnquoteExtract builds: ->>'' +func (mysqlDialect) JSONUnquoteExtract(col, path string) string { + return fmt.Sprintf("%s->>'%s'", col, path) +} + +// JSONBuildObject builds: JSON_OBJECT(, , ...) +func (mysqlDialect) JSONBuildObject(keyVals ...string) string { + return fmt.Sprintf("JSON_OBJECT(%s)", strings.Join(keyVals, ", ")) +} + +// FindInSet builds: FIND_IN_SET(, ) +func (mysqlDialect) FindInSet(needle, col string) string { + return fmt.Sprintf("FIND_IN_SET(%s, %s)", needle, col) +} + +// FullTextMatch builds: MATCH() AGAINST ( IN BOOLEAN MODE) +func (mysqlDialect) FullTextMatch(cols []string, query string) string { + return fmt.Sprintf("MATCH(%s) AGAINST (%s IN BOOLEAN MODE)", strings.Join(cols, ", "), query) +} + +// RegexpMatch builds: REGEXP +func (mysqlDialect) RegexpMatch(col, pattern string) string { + return fmt.Sprintf("%s REGEXP %s", col, pattern) +} + +// GoquDialect returns the goqu MySQL dialect wrapper. +func (mysqlDialect) GoquDialect() goqu.DialectWrapper { + return goqu.Dialect("mysql") +} + +// IsDuplicate delegates to the package-level IsDuplicate in errors.go. +func (mysqlDialect) IsDuplicate(err error) bool { + return IsDuplicate(err) +} + +// IsForeignKey delegates to the package-level isMySQLForeignKey in errors.go. +func (mysqlDialect) IsForeignKey(err error) bool { + return isMySQLForeignKey(err) +} + +// IsReadOnly delegates to common_mysql.IsReadOnlyError. +func (mysqlDialect) IsReadOnly(err error) bool { + return common_mysql.IsReadOnlyError(err) +} + +// IsBadConnection delegates to the package-level isBadConnection in errors.go. +func (mysqlDialect) IsBadConnection(err error) bool { + return isBadConnection(err) +} + +func (mysqlDialect) ReturningID() string { return "" } + +func (mysqlDialect) IsPostgres() bool { return false } diff --git a/server/datastore/mysql/dialect_mysql_test.go b/server/datastore/mysql/dialect_mysql_test.go new file mode 100644 index 00000000000..b140c044ebc --- /dev/null +++ b/server/datastore/mysql/dialect_mysql_test.go @@ -0,0 +1,67 @@ +package mysql + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMysqlDialectSQL(t *testing.T) { + d := mysqlDialect{} + + t.Run("InsertIgnoreInto", func(t *testing.T) { + assert.Equal(t, "INSERT IGNORE INTO", d.InsertIgnoreInto()) + }) + + t.Run("ReplaceInto", func(t *testing.T) { + assert.Equal(t, "REPLACE INTO", d.ReplaceInto()) + }) + + t.Run("OnDuplicateKey", func(t *testing.T) { + got := d.OnDuplicateKey("id", "name=VALUES(name), updated_at=NOW()") + assert.Equal(t, "ON DUPLICATE KEY UPDATE name=VALUES(name), updated_at=NOW()", got) + }) + + t.Run("OnConflictDoNothing", func(t *testing.T) { + assert.Equal(t, "", d.OnConflictDoNothing("id")) + }) + + t.Run("GroupConcat", func(t *testing.T) { + assert.Equal(t, "GROUP_CONCAT(x SEPARATOR ',')", d.GroupConcat("x", ",")) + assert.Equal(t, "GROUP_CONCAT(DISTINCT v.col SEPARATOR ',')", d.GroupConcat("DISTINCT v.col", ",")) + }) + + t.Run("JSONExtract", func(t *testing.T) { + assert.Equal(t, "JSON_EXTRACT(col, '$.path')", d.JSONExtract("col", "$.path")) + }) + + t.Run("JSONUnquoteExtract", func(t *testing.T) { + assert.Equal(t, "col->>'$.path'", d.JSONUnquoteExtract("col", "$.path")) + }) + + t.Run("JSONBuildObject", func(t *testing.T) { + assert.Equal(t, "JSON_OBJECT('k1', v1, 'k2', v2)", d.JSONBuildObject("'k1'", "v1", "'k2'", "v2")) + }) + + t.Run("FindInSet", func(t *testing.T) { + assert.Equal(t, "FIND_IN_SET(?, platforms)", d.FindInSet("?", "platforms")) + }) + + t.Run("FullTextMatch", func(t *testing.T) { + assert.Equal(t, "MATCH(l.name) AGAINST (? IN BOOLEAN MODE)", d.FullTextMatch([]string{"l.name"}, "?")) + }) + + t.Run("RegexpMatch", func(t *testing.T) { + assert.Equal(t, "s.name REGEXP ?", d.RegexpMatch("s.name", "?")) + }) + + t.Run("JSONAgg", func(t *testing.T) { + assert.Equal(t, "JSON_ARRAYAGG(x)", d.JSONAgg("x")) + }) + + t.Run("GoquDialect", func(t *testing.T) { + // Verify it returns a valid goqu dialect (not nil/panic) + gd := d.GoquDialect() + assert.NotNil(t, gd) + }) +} diff --git a/server/datastore/mysql/dialect_postgres.go b/server/datastore/mysql/dialect_postgres.go new file mode 100644 index 00000000000..21e9e7f142c --- /dev/null +++ b/server/datastore/mysql/dialect_postgres.go @@ -0,0 +1,175 @@ +// dialect_postgres.go implements DialectHelper for PostgreSQL. + +package mysql + +import ( + "fmt" + "regexp" + "strings" + + "github.com/doug-martin/goqu/v9" + _ "github.com/doug-martin/goqu/v9/dialect/postgres" // register postgres dialect + pg "github.com/fleetdm/fleet/v4/server/platform/postgres" +) + +// postgresDialect implements DialectHelper for PostgreSQL. +type postgresDialect struct{} + +// Compile-time assertion that postgresDialect satisfies DialectHelper. +var _ DialectHelper = postgresDialect{} + +// InsertIgnoreInto returns "INSERT INTO". +// PostgreSQL achieves ignore semantics via ON CONFLICT ... DO NOTHING appended by the caller. +func (postgresDialect) InsertIgnoreInto() string { return "INSERT INTO" } + +// ReplaceInto returns "INSERT INTO". +// PostgreSQL achieves replace semantics via ON CONFLICT ... DO UPDATE SET appended by the caller. +func (postgresDialect) ReplaceInto() string { return "INSERT INTO" } + +// valuesPattern matches MySQL VALUES(`col`) or VALUES(col) in ON DUPLICATE KEY UPDATE clauses. +var valuesPattern = regexp.MustCompile("VALUES\\(`?([^`)]+)`?\\)") + +// lastInsertIDPattern matches id=LAST_INSERT_ID(id) assignments in ON DUPLICATE KEY UPDATE clauses. +// This MySQL trick returns the existing row's ID on conflict; PG uses RETURNING id instead. +var lastInsertIDPattern = regexp.MustCompile(`(?:,\s*)?id\s*=\s*LAST_INSERT_ID\(id\)(?:\s*,)?`) + +// stripLastInsertID removes id=LAST_INSERT_ID(id) from an ON DUPLICATE KEY UPDATE clause. +func stripLastInsertID(clause string) string { + result := lastInsertIDPattern.ReplaceAllString(clause, "") + return strings.Trim(result, ", ") +} + +// translateValuesToExcluded rewrites MySQL VALUES(col) references to PostgreSQL EXCLUDED.col. +// +// VALUES(name) → EXCLUDED.name +// VALUES(`name`) → EXCLUDED.name +func translateValuesToExcluded(clause string) string { + return valuesPattern.ReplaceAllString(clause, "EXCLUDED.$1") +} + +// OnDuplicateKey returns: ON CONFLICT () DO UPDATE SET +// The updateClause uses MySQL syntax; VALUES(col) is translated to EXCLUDED.col. +// If the clause contains id=LAST_INSERT_ID(id), it is stripped (PG uses RETURNING id). +// If stripping leaves an empty clause, a no-op update on the first conflict column is used +// so that RETURNING id still works. +func (postgresDialect) OnDuplicateKey(conflictTarget, updateClause string) string { + cleaned := stripLastInsertID(updateClause) + if strings.TrimSpace(cleaned) == "" { + // No-op update: set the first conflict column to itself so RETURNING id works. + firstCol := strings.SplitN(conflictTarget, ",", 2)[0] + firstCol = strings.TrimSpace(firstCol) + return "ON CONFLICT (" + conflictTarget + ") DO UPDATE SET " + firstCol + " = EXCLUDED." + firstCol + } + return "ON CONFLICT (" + conflictTarget + ") DO UPDATE SET " + translateValuesToExcluded(cleaned) +} + +// OnConflictDoNothing returns: ON CONFLICT () DO NOTHING +func (postgresDialect) OnConflictDoNothing(conflictTarget string) string { + return " ON CONFLICT (" + conflictTarget + ") DO NOTHING" +} + +// GroupConcat builds: STRING_AGG(::text, '') +func (postgresDialect) GroupConcat(expr, separator string) string { + return fmt.Sprintf("STRING_AGG(%s::text, '%s')", expr, separator) +} + +// JSONAgg builds: jsonb_agg() — uses jsonb_agg for PG jsonb compatibility +func (postgresDialect) JSONAgg(expr string) string { + return fmt.Sprintf("jsonb_agg(%s)", expr) +} + +// mysqlPathToPGChain converts a MySQL JSON path ($.key1.key2) to a chain of +// PostgreSQL -> operators: col->'key1'->'key2'. +// For a single-level path like $.path, it returns col->'path'. +// The final operator is determined by the extract parameter: +// +// extract=false → all segments use -> (returns JSON) +// extract=true → last segment uses ->> (returns text) +func mysqlPathToPGChain(col, path string, extractText bool) string { + // Strip $. prefix + path = strings.TrimPrefix(path, "$.") + // Remove surrounding double quotes + path = strings.Trim(path, `"`) + + // Split on . to get path segments + segments := strings.Split(path, ".") + if len(segments) == 0 { + return col + } + + var b strings.Builder + b.WriteString(col) + for i, seg := range segments { + if extractText && i == len(segments)-1 { + b.WriteString("->>'") + } else { + b.WriteString("->'") + } + b.WriteString(seg) + b.WriteByte('\'') + } + return b.String() +} + +// JSONExtract builds a PG JSON traversal returning JSON (uses -> for all levels). +// +// MySQL: JSON_EXTRACT(col, '$.mdm.setting') → PG: col->'mdm'->'setting' +// MySQL: JSON_EXTRACT(col, '$.path') → PG: col->'path' +func (postgresDialect) JSONExtract(col, path string) string { + return mysqlPathToPGChain(col, path, false) +} + +// JSONUnquoteExtract builds a PG JSON traversal returning text (last level uses ->>). +// +// MySQL: col->>'$.mdm.setting' → PG: col->'mdm'->>'setting' +// MySQL: col->>'$.path' → PG: col->>'path' +func (postgresDialect) JSONUnquoteExtract(col, path string) string { + return mysqlPathToPGChain(col, path, true) +} + +// JSONBuildObject builds: jsonb_build_object(, , ...) +func (postgresDialect) JSONBuildObject(keyVals ...string) string { + return fmt.Sprintf("jsonb_build_object(%s)", strings.Join(keyVals, ", ")) +} + +// FindInSet builds: = ANY(string_to_array(, ',')) +func (postgresDialect) FindInSet(needle, col string) string { + return fmt.Sprintf("%s = ANY(string_to_array(%s, ','))", needle, col) +} + +// FullTextMatch builds: to_tsvector('english', ) @@ plainto_tsquery('english', ) +// PostgreSQL's to_tsvector takes a single column expression. +func (postgresDialect) FullTextMatch(cols []string, query string) string { + return fmt.Sprintf("to_tsvector('english', %s) @@ plainto_tsquery('english', %s)", cols[0], query) +} + +// RegexpMatch builds: ~ +func (postgresDialect) RegexpMatch(col, pattern string) string { + return fmt.Sprintf("%s ~ %s", col, pattern) +} + +// GoquDialect returns the goqu PostgreSQL dialect wrapper. +func (postgresDialect) GoquDialect() goqu.DialectWrapper { + return goqu.Dialect("postgres") +} + +// --- Error classification --- +// +// Delegates to server/platform/postgres which uses proper pgx/pq interface +// matching via SQLSTATE codes. + +// IsDuplicate returns true if err is a unique-constraint violation (SQLSTATE 23505). +func (postgresDialect) IsDuplicate(err error) bool { return pg.IsDuplicate(err) } + +// IsForeignKey returns true if err is a foreign-key constraint violation (SQLSTATE 23503). +func (postgresDialect) IsForeignKey(err error) bool { return pg.IsForeignKey(err) } + +// IsReadOnly returns true if err indicates a read-only transaction (SQLSTATE 25006). +func (postgresDialect) IsReadOnly(err error) bool { return pg.IsReadOnly(err) } + +// IsBadConnection returns true if err is a connection-level error. +func (postgresDialect) IsBadConnection(err error) bool { return pg.IsBadConnection(err) } + +func (postgresDialect) ReturningID() string { return " RETURNING id" } + +func (postgresDialect) IsPostgres() bool { return true } diff --git a/server/datastore/mysql/dialect_postgres_test.go b/server/datastore/mysql/dialect_postgres_test.go new file mode 100644 index 00000000000..7274e0d8592 --- /dev/null +++ b/server/datastore/mysql/dialect_postgres_test.go @@ -0,0 +1,115 @@ +package mysql + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPostgresDialectSQL(t *testing.T) { + d := postgresDialect{} + + t.Run("InsertIgnoreInto", func(t *testing.T) { + assert.Equal(t, "INSERT INTO", d.InsertIgnoreInto()) + }) + + t.Run("ReplaceInto", func(t *testing.T) { + assert.Equal(t, "INSERT INTO", d.ReplaceInto()) + }) + + t.Run("OnDuplicateKey", func(t *testing.T) { + got := d.OnDuplicateKey("id", "name=VALUES(name), updated_at=NOW()") + assert.Equal(t, "ON CONFLICT (id) DO UPDATE SET name=EXCLUDED.name, updated_at=NOW()", got) + }) + + t.Run("OnDuplicateKey_backtick_quoted", func(t *testing.T) { + got := d.OnDuplicateKey("id", "`name`=VALUES(`name`)") + assert.Equal(t, "ON CONFLICT (id) DO UPDATE SET `name`=EXCLUDED.name", got) + }) + + t.Run("OnConflictDoNothing", func(t *testing.T) { + assert.Equal(t, " ON CONFLICT (host_id, label_id) DO NOTHING", d.OnConflictDoNothing("host_id, label_id")) + }) + + t.Run("GroupConcat", func(t *testing.T) { + assert.Equal(t, "STRING_AGG(x::text, ',')", d.GroupConcat("x", ",")) + }) + + t.Run("JSONExtract_dollar_dot", func(t *testing.T) { + assert.Equal(t, "col->'path'", d.JSONExtract("col", "$.path")) + }) + + t.Run("JSONExtract_nested", func(t *testing.T) { + assert.Equal(t, "t.config->'mdm'->'enable_recovery_lock_password'", d.JSONExtract("t.config", "$.mdm.enable_recovery_lock_password")) + }) + + t.Run("JSONUnquoteExtract", func(t *testing.T) { + assert.Equal(t, "col->>'path'", d.JSONUnquoteExtract("col", "$.path")) + }) + + t.Run("JSONBuildObject", func(t *testing.T) { + assert.Equal(t, "jsonb_build_object('k1', v1)", d.JSONBuildObject("'k1'", "v1")) + }) + + t.Run("FindInSet", func(t *testing.T) { + assert.Equal(t, "? = ANY(string_to_array(platforms, ','))", d.FindInSet("?", "platforms")) + }) + + t.Run("FullTextMatch", func(t *testing.T) { + assert.Equal(t, "to_tsvector('english', l.name) @@ plainto_tsquery('english', ?)", d.FullTextMatch([]string{"l.name"}, "?")) + }) + + t.Run("RegexpMatch", func(t *testing.T) { + assert.Equal(t, "s.name ~ ?", d.RegexpMatch("s.name", "?")) + }) + + t.Run("JSONAgg", func(t *testing.T) { + assert.Equal(t, "jsonb_agg(x)", d.JSONAgg("x")) + }) + + t.Run("GoquDialect", func(t *testing.T) { + gd := d.GoquDialect() + assert.NotNil(t, gd) + }) +} + +func TestTranslateValuesToExcluded(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"name=VALUES(name)", "name=EXCLUDED.name"}, + {"name=VALUES(name), age=VALUES(age)", "name=EXCLUDED.name, age=EXCLUDED.age"}, + {"`name`=VALUES(`name`)", "`name`=EXCLUDED.name"}, + {"col = col + VALUES(col)", "col = col + EXCLUDED.col"}, + {"updated_at=NOW()", "updated_at=NOW()"}, + {"iteration = iteration + 1", "iteration = iteration + 1"}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + assert.Equal(t, tt.expected, translateValuesToExcluded(tt.input)) + }) + } +} + +func TestMysqlPathToPGChain(t *testing.T) { + tests := []struct { + col, path string + extractText bool + expected string + }{ + {"col", "$.path", false, "col->'path'"}, + {"col", "$.path", true, "col->>'path'"}, + {"t.config", "$.mdm.enable_recovery_lock_password", false, "t.config->'mdm'->'enable_recovery_lock_password'"}, + {"t.config", "$.mdm.enable_recovery_lock_password", true, "t.config->'mdm'->>'enable_recovery_lock_password'"}, + {"col", "$.\"quoted\"", false, "col->'quoted'"}, + {"col", "path", false, "col->'path'"}, + } + + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + assert.Equal(t, tt.expected, mysqlPathToPGChain(tt.col, tt.path, tt.extractText)) + }) + } +} diff --git a/server/goose/dialect.go b/server/goose/dialect.go index bfa5f879cb9..763b937b5c0 100644 --- a/server/goose/dialect.go +++ b/server/goose/dialect.go @@ -11,6 +11,10 @@ type SqlDialect interface { createVersionTableSql(name string) string // sql string to create the goose_db_version table insertVersionSql(name string) string // sql string to insert the initial version table row dbVersionQuery(db *sql.DB, name string) (*sql.Rows, error) + + // DriverName returns the driver name for this dialect ("mysql", "postgres", "sqlite3"). + // Used by the migration runner to select dialect-specific UpFnMySQL/UpFnPG functions. + DriverName() string } func GetDialect() SqlDialect { @@ -42,8 +46,10 @@ func SetDialect(d string) error { type PostgresDialect struct{} +func (PostgresDialect) DriverName() string { return "postgres" } + func (pg PostgresDialect) createVersionTableSql(name string) string { - return `CREATE TABLE ` + name + ` ( + return `CREATE TABLE IF NOT EXISTS ` + name + ` ( id serial NOT NULL, version_id bigint NOT NULL, is_applied boolean NOT NULL, @@ -72,8 +78,10 @@ func (pg PostgresDialect) dbVersionQuery(db *sql.DB, name string) (*sql.Rows, er type MySqlDialect struct{} +func (MySqlDialect) DriverName() string { return "mysql" } + func (m MySqlDialect) createVersionTableSql(name string) string { - return `CREATE TABLE ` + name + ` ( + return `CREATE TABLE IF NOT EXISTS ` + name + ` ( id serial NOT NULL, version_id bigint NOT NULL, is_applied boolean NOT NULL, @@ -102,6 +110,8 @@ func (m MySqlDialect) dbVersionQuery(db *sql.DB, name string) (*sql.Rows, error) type Sqlite3Dialect struct{} +func (Sqlite3Dialect) DriverName() string { return "sqlite3" } + func (m Sqlite3Dialect) createVersionTableSql(name string) string { return `CREATE TABLE ` + name + ` ( id INTEGER PRIMARY KEY AUTOINCREMENT, diff --git a/server/goose/migrate.go b/server/goose/migrate.go index ee8d3504fa4..f672b63bdd8 100644 --- a/server/goose/migrate.go +++ b/server/goose/migrate.go @@ -89,6 +89,25 @@ func AddMigration(up func(*sql.Tx) error, down func(*sql.Tx) error) { globalGoose.Migrations = append(globalGoose.Migrations, migration) } +// AddDualDialectMigration adds a migration with dialect-specific up/down functions. +// Use this for migrations where MySQL and PostgreSQL need different DDL. +// Pass nil for any function that should be a no-op for that dialect. +func (c *Client) AddDualDialectMigration(upMySQL, downMySQL, upPG, downPG func(*sql.Tx) error) { + _, filename, _, _ := runtime.Caller(1) + v, _ := NumericComponent(filename) + migration := &Migration{ + Version: v, + Next: -1, + Previous: -1, + Source: filename, + UpFnMySQL: upMySQL, + DownFnMySQL: downMySQL, + UpFnPG: upPG, + DownFnPG: downPG, + } + c.Migrations = append(c.Migrations, migration) +} + // collect all the valid looking migration scripts in the // migrations folder and go func registry, and key them by version func (c *Client) collectMigrations(dirpath string, current, target int64) (Migrations, error) { diff --git a/server/goose/migrate_test.go b/server/goose/migrate_test.go index fb64aad6408..6589e59c9e8 100644 --- a/server/goose/migrate_test.go +++ b/server/goose/migrate_test.go @@ -1,6 +1,9 @@ package goose -import "testing" +import ( + "database/sql" + "testing" +) func newMigration(v int64, src string) *Migration { return &Migration{Version: v, Previous: -1, Next: -1, Source: src} @@ -55,3 +58,69 @@ func validateMigrationSort(t *testing.T, ms Migrations, sorted []int64) { t.Log(ms) } + +func TestMigrationSelectFn(t *testing.T) { + generic := func(*sql.Tx) error { return nil } + mysqlFn := func(*sql.Tx) error { return nil } + pgFn := func(*sql.Tx) error { return nil } + + t.Run("generic only", func(t *testing.T) { + m := &Migration{UpFn: generic, DownFn: generic} + if m.selectFn("mysql", true) == nil { + t.Error("expected generic up for mysql") + } + if m.selectFn("postgres", true) == nil { + t.Error("expected generic up for postgres") + } + }) + + t.Run("mysql specific takes precedence", func(t *testing.T) { + m := &Migration{UpFn: generic, UpFnMySQL: mysqlFn} + // MySQL should get mysqlFn, not generic + fn := m.selectFn("mysql", true) + if fn == nil { + t.Fatal("expected non-nil fn for mysql") + } + // Postgres should fall back to generic + fn = m.selectFn("postgres", true) + if fn == nil { + t.Fatal("expected non-nil fn for postgres") + } + }) + + t.Run("pg specific takes precedence", func(t *testing.T) { + m := &Migration{UpFn: generic, UpFnPG: pgFn} + fn := m.selectFn("postgres", true) + if fn == nil { + t.Fatal("expected non-nil fn for postgres") + } + fn = m.selectFn("mysql", true) + if fn == nil { + t.Fatal("expected non-nil fn for mysql fallback to generic") + } + }) + + t.Run("dual dialect no generic", func(t *testing.T) { + m := &Migration{UpFnMySQL: mysqlFn, UpFnPG: pgFn} + if m.selectFn("mysql", true) == nil { + t.Error("expected mysql fn") + } + if m.selectFn("postgres", true) == nil { + t.Error("expected pg fn") + } + // unknown driver falls back to nil generic + if m.selectFn("sqlite3", true) != nil { + t.Error("expected nil for sqlite3 with no generic") + } + }) + + t.Run("down direction", func(t *testing.T) { + m := &Migration{DownFn: generic, DownFnMySQL: mysqlFn} + if m.selectFn("mysql", false) == nil { + t.Error("expected mysql down fn") + } + if m.selectFn("postgres", false) == nil { + t.Error("expected generic down for postgres") + } + }) +} diff --git a/server/goose/migration.go b/server/goose/migration.go index b3c2c55f7ac..5e70ee8b24e 100644 --- a/server/goose/migration.go +++ b/server/goose/migration.go @@ -24,8 +24,18 @@ type Migration struct { Next int64 // next version, or -1 if none Previous int64 // previous version, -1 if none Source string // path to .sql script - UpFn func(*sql.Tx) error // Up go migration function - DownFn func(*sql.Tx) error // Down go migration function + UpFn func(*sql.Tx) error // Up go migration function (dialect-agnostic fallback) + DownFn func(*sql.Tx) error // Down go migration function (dialect-agnostic fallback) + + // UpFnMySQL and DownFnMySQL are MySQL-specific migration functions. + // When set, they take precedence over UpFn/DownFn for MySQL databases. + UpFnMySQL func(*sql.Tx) error + DownFnMySQL func(*sql.Tx) error + + // UpFnPG and DownFnPG are PostgreSQL-specific migration functions. + // When set, they take precedence over UpFn/DownFn for PostgreSQL databases. + UpFnPG func(*sql.Tx) error + DownFnPG func(*sql.Tx) error } const ( @@ -33,6 +43,36 @@ const ( migrateDown = !migrateUp ) +// selectFn returns the appropriate migration function for the given driver and direction. +// It prefers dialect-specific functions (UpFnMySQL, UpFnPG) over the generic UpFn/DownFn. +func (m *Migration) selectFn(driver string, direction bool) func(*sql.Tx) error { + if direction { // up + switch driver { + case "mysql": + if m.UpFnMySQL != nil { + return m.UpFnMySQL + } + case "postgres": + if m.UpFnPG != nil { + return m.UpFnPG + } + } + return m.UpFn + } + // down + switch driver { + case "mysql": + if m.DownFnMySQL != nil { + return m.DownFnMySQL + } + case "postgres": + if m.DownFnPG != nil { + return m.DownFnPG + } + } + return m.DownFn +} + func (m *Migration) String() string { return fmt.Sprint(m.Source) } @@ -53,10 +93,7 @@ func (c *Client) runMigration(db *sql.DB, m *Migration, direction bool) error { log.Fatal("db.Begin: ", err) } - fn := m.UpFn - if !direction { - fn = m.DownFn - } + fn := m.selectFn(c.Dialect.DriverName(), direction) if fn != nil { if err := fn(tx); err != nil { tx.Rollback() //nolint:errcheck diff --git a/server/platform/postgres/common.go b/server/platform/postgres/common.go new file mode 100644 index 00000000000..137b2285c49 --- /dev/null +++ b/server/platform/postgres/common.go @@ -0,0 +1,31 @@ +package postgres + +import ( + "fmt" + + "github.com/jmoiron/sqlx" +) + +// NewDB opens a PostgreSQL database connection using the standard database/sql +// interface via the pgx stdlib driver. The dsn should be a PostgreSQL connection +// string (e.g., "postgres://user:pass@host:5432/dbname?sslmode=disable"). +// +// Callers should register the pgx stdlib driver before calling this function: +// +// import _ "github.com/jackc/pgx/v5/stdlib" +func NewDB(dsn string, maxOpenConns, maxIdleConns int) (*sqlx.DB, error) { + db, err := sqlx.Open("pgx", dsn) + if err != nil { + return nil, fmt.Errorf("open postgres connection: %w", err) + } + + db.SetMaxOpenConns(maxOpenConns) + db.SetMaxIdleConns(maxIdleConns) + + if err := db.Ping(); err != nil { + db.Close() + return nil, fmt.Errorf("ping postgres: %w", err) + } + + return db, nil +} diff --git a/server/platform/postgres/errors.go b/server/platform/postgres/errors.go new file mode 100644 index 00000000000..d7bf6969f14 --- /dev/null +++ b/server/platform/postgres/errors.go @@ -0,0 +1,114 @@ +// Package postgres provides PostgreSQL-specific utilities for Fleet's datastore layer. +package postgres + +import ( + "database/sql/driver" + "errors" + "io" + "net" + "os" + "strings" + "syscall" +) + +// PostgreSQL error codes (from SQLSTATE). +// See: https://www.postgresql.org/docs/current/errcodes-appendix.html +const ( + // Class 23 — Integrity Constraint Violation + codeUniqueViolation = "23505" + codeForeignKeyViolation = "23503" + + // Class 25 — Invalid Transaction State + codeReadOnlySQLTransaction = "25006" + + // Class 08 — Connection Exception + codeConnectionException = "08000" + codeConnectionFailure = "08006" + codeProtocolViolation = "08P01" + codeSQLClientUnableToEst = "08001" +) + +// IsDuplicate returns true if the error is a PostgreSQL unique_violation (23505). +func IsDuplicate(err error) bool { + return hasErrorCode(err, codeUniqueViolation) +} + +// IsForeignKey returns true if the error is a PostgreSQL foreign_key_violation (23503). +func IsForeignKey(err error) bool { + return hasErrorCode(err, codeForeignKeyViolation) +} + +// IsReadOnly returns true if the error indicates a read-only transaction (25006). +func IsReadOnly(err error) bool { + return hasErrorCode(err, codeReadOnlySQLTransaction) +} + +// IsBadConnection returns true if the error is a connection-level error +// that justifies retrying on a new connection. +func IsBadConnection(err error) bool { + if err == nil { + return false + } + + // Standard database/sql connection errors. + if errors.Is(err, driver.ErrBadConn) || + errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, io.EOF) || + errors.Is(err, syscall.ECONNREFUSED) || + errors.Is(err, syscall.ECONNRESET) || + errors.Is(err, syscall.ENETUNREACH) || + errors.Is(err, syscall.ETIMEDOUT) { + return true + } + + // PostgreSQL connection exception codes. + if hasErrorCode(err, codeConnectionException) || + hasErrorCode(err, codeConnectionFailure) || + hasErrorCode(err, codeProtocolViolation) || + hasErrorCode(err, codeSQLClientUnableToEst) { + return true + } + + // OS-level network errors. + var se *os.SyscallError + if errors.As(err, &se) { + return errors.Is(se.Err, syscall.ECONNRESET) || errors.Is(se.Err, syscall.EPIPE) + } + + var netErr *net.OpError + if errors.As(err, &netErr) { + return true + } + + return false +} + +// hasErrorCode checks if the error (or any wrapped error) contains the given +// PostgreSQL SQLSTATE code. This works with any error type that implements +// a Code() or SQLState() method, including pgx and lib/pq errors. +func hasErrorCode(err error, code string) bool { + if err == nil { + return false + } + + // Check for pgx-style error (implements Code() string). + type pgxError interface { + Code() string + } + var pgxErr pgxError + if errors.As(err, &pgxErr) { + return string(pgxErr.Code()) == code + } + + // Check for lib/pq-style error (has Code field via the pq.Error type). + type pqError interface { + Get(byte) string + } + var pqErr pqError + if errors.As(err, &pqErr) { + return pqErr.Get('C') == code // 'C' = Code field + } + + // Fallback: check error string for the code (defensive). + return strings.Contains(err.Error(), code) +} diff --git a/server/platform/postgres/rebind_driver.go b/server/platform/postgres/rebind_driver.go new file mode 100644 index 00000000000..eab5e690827 --- /dev/null +++ b/server/platform/postgres/rebind_driver.go @@ -0,0 +1,1590 @@ +// Package postgres provides a MySQL-to-PostgreSQL SQL rebind driver for Fleet. +// It wraps pgx/v5 to automatically translate MySQL-dialect SQL to PostgreSQL, +// including placeholder conversion (? → $N), function rewrites (IF → CASE WHEN, +// JSON_OBJECT → jsonb_build_object, etc.), and type fixes (boolean = integer). +// Register with: sql.Register("pgx-rebind", &rebindDriver{}) +package postgres + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "regexp" + "strings" + "time" + "unicode/utf8" + + "github.com/jackc/pgx/v5/stdlib" +) + +// Pre-compiled regexes used in rebindQuery to avoid per-query compilation overhead. +var ( + reUUIDBinUpper = regexp.MustCompile(`UUID_TO_BIN\(UUID\(\),\s*true\)`) + reUUIDBinLower = regexp.MustCompile(`UUID_TO_BIN\(uuid\(\),\s*true\)`) + reUUIDBinTrue = regexp.MustCompile(`UUID_TO_BIN\(([^,)]+),\s*true\)`) + reUUIDBin = regexp.MustCompile(`UUID_TO_BIN\(([^,)]+)\)`) + reUUID = regexp.MustCompile(`(?i)\bUUID\(\)`) + reBinToUUIDTrue = regexp.MustCompile(`BIN_TO_UUID\(([^,)]+),\s*true\)`) + reBinToUUID = regexp.MustCompile(`BIN_TO_UUID\(([^,)]+)\)`) + reTimeDiff = regexp.MustCompile(`TIMEDIFF\(([^,]+),\s*([^)]+)\)`) + reTimeToSec = regexp.MustCompile(`TIME_TO_SEC\(([^)]+)\)`) + reFromDual = regexp.MustCompile(`(?i)\s+FROM\s+DUAL\b`) + reSeparator = regexp.MustCompile(`(?i)\bSEPARATOR\s+'([^']*)'`) + reTimestamp = regexp.MustCompile(`\bTIMESTAMP\(([^)]+)\)`) + reMaxDenylisted = regexp.MustCompile(`MAX\(([^)]*\.denylisted)\)`) + // MAX(prof_*) columns from boolean subqueries (android/apple MDM profile status aggregation) + reMaxBooleanCols = regexp.MustCompile(`MAX\(((?:prof|fv|rl|decl)_(?:pending|failed|verifying|verified)|android_prof_(?:pending|failed|verifying|verified))\)`) + reLimitTrailing = regexp.MustCompile(`(?i)\s+LIMIT\s+\d+\s*$`) + reJSONExtractFunc = regexp.MustCompile(`JSON_EXTRACT\((\w+),\s*(\?|'[^']*')\)`) + reJSONPath = regexp.MustCompile(`->>?'\$\.[^']*'`) + reTimestampDiff = regexp.MustCompile(`(?i)TIMESTAMPDIFF\(\s*SECOND\s*,\s*(.+?)\s*,\s*(.+?)\s*\)`) + reNormalizeDuplicateKey = regexp.MustCompile(`(?i)ON\s+DUPLICATE\s+KEY\s+UPDATE`) + // MySQL: INSERT INTO table () VALUES () — empty column/value lists for auto-increment-only inserts + reEmptyValues = regexp.MustCompile(`(?i)(INSERT\s+INTO\s+\S+\s+)\(\s*\)\s*VALUES\s*\(\s*\)`) + // PG can't infer $N type in interval arithmetic; cast to timestamptz + reParamBeforeInterval = regexp.MustCompile(`(\$\d+)\s+([-+*]\s*INTERVAL\b)`) + // JSON boolean comparison: MySQL ->> on JSON true returns '1', PG returns 'true'. + // Match: COALESCE(, '0') = '1' → COALESCE(, '0') IN ('1', 'true') + reJSONBoolCoalesce = regexp.MustCompile(`COALESCE\(([^)]+->>'[^']+'),\s*'0'\)\s*=\s*'1'`) + + // Per-unit INTERVAL regexes (SECOND, MINUTE, HOUR, DAY) + reIntervalLiteral = map[string]*regexp.Regexp{} + reIntervalPlaceholder = map[string]*regexp.Regexp{} +) + +func init() { + for _, unit := range []string{"SECOND", "MINUTE", "HOUR", "DAY"} { + reIntervalLiteral[unit] = regexp.MustCompile(`INTERVAL\s+(\d+)\s+` + unit) + reIntervalPlaceholder[unit] = regexp.MustCompile(`INTERVAL\s+(\?)\s+` + unit) + } +} + +func init() { + // Register "pgx-rebind" as a wrapper driver that auto-rewrites ? → $N. + // This allows MySQL-style ? placeholders to work transparently with PG. + sql.Register("pgx-rebind", &rebindDriver{}) +} + +type rebindDriver struct{} + +func (d *rebindDriver) Open(dsn string) (driver.Conn, error) { + connector, err := stdlib.GetDefaultDriver().(*stdlib.Driver).OpenConnector(dsn) + if err != nil { + return nil, err + } + conn, err := connector.Connect(context.Background()) + if err != nil { + return nil, err + } + return &rebindConn{Conn: conn}, nil +} + +func (d *rebindDriver) OpenConnector(dsn string) (driver.Connector, error) { + base, err := stdlib.GetDefaultDriver().(*stdlib.Driver).OpenConnector(dsn) + if err != nil { + return nil, err + } + return &rebindConnector{base: base}, nil +} + +type rebindConnector struct { + base driver.Connector +} + +func (c *rebindConnector) Connect(ctx context.Context) (driver.Conn, error) { + conn, err := c.base.Connect(ctx) + if err != nil { + return nil, err + } + return &rebindConn{Conn: conn}, nil +} + +func (c *rebindConnector) Driver() driver.Driver { + return &rebindDriver{} +} + +type rebindConn struct { + driver.Conn +} + +// BeginTx delegates to the underlying connection's ConnBeginTx interface, +// enabling support for non-default isolation levels and read-only transactions. +func (c *rebindConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if cbt, ok := c.Conn.(driver.ConnBeginTx); ok { + return cbt.BeginTx(ctx, opts) + } + // Fall back to Begin() if the underlying conn doesn't support BeginTx + return c.Conn.Begin() //nolint:staticcheck // fallback for drivers without ConnBeginTx +} + +// rebindQuery converts MySQL-specific SQL to PostgreSQL. +// It handles: ? → $N placeholders, JSON_OBJECT → jsonb_build_object, +// DATE_ADD → PG interval arithmetic, INTERVAL N SECOND/MINUTE/etc. +func rebindQuery(query string) string { + // Skip rewriting PL/pgSQL function bodies and DDL that shouldn't be modified + if strings.Contains(query, "$$") || strings.HasPrefix(strings.TrimSpace(strings.ToUpper(query)), "CREATE TRIGGER") { + return query + } + + // INSERT IGNORE INTO → INSERT INTO ... ON CONFLICT DO NOTHING + hasInsertIgnore := false + if strings.Contains(query, "INSERT IGNORE") { + query = strings.Replace(query, "INSERT IGNORE INTO", "INSERT INTO", 1) + query = strings.Replace(query, "INSERT IGNORE", "INSERT", 1) + hasInsertIgnore = true + } + + // MySQL: INSERT INTO t () VALUES () → PG: INSERT INTO t DEFAULT VALUES + // MySQL allows empty column/value lists to insert a row with all defaults; PG does not. + query = reEmptyValues.ReplaceAllString(query, "${1}DEFAULT VALUES") + + // Replace MySQL-specific functions with PG equivalents + // NOW(6) / CURRENT_TIMESTAMP(6) → NOW() / CURRENT_TIMESTAMP (PG already returns microsecond precision) + query = strings.ReplaceAll(query, "NOW(6)", "NOW()") + query = strings.ReplaceAll(query, "CURRENT_TIMESTAMP(6)", "CURRENT_TIMESTAMP") + // CURRENT_TIMESTAMP() → CURRENT_TIMESTAMP (PG doesn't use parens) + query = strings.ReplaceAll(query, "CURRENT_TIMESTAMP()", "CURRENT_TIMESTAMP") + // MD5() → md5() (PG uses lowercase) + query = strings.ReplaceAll(query, "MD5(", "md5(") + // JSON_EXTRACT(col, expr) → (col->regexp_replace(expr, '^\$\.?"?', '')) + // MySQL JSON_EXTRACT uses $.path syntax; PG -> operator uses plain key names. + // The regexp_replace strips the $. prefix and optional quotes at runtime. + if strings.Contains(query, "JSON_EXTRACT(") { + query = rewriteJSONExtractFunc(query) + } + // JSON_OBJECT → jsonb_build_object, then cast placeholder args to text + // (PG's jsonb_build_object has VARIADIC "any" so it can't infer $N types) + query = strings.ReplaceAll(query, "JSON_OBJECT(", "jsonb_build_object(") + query = castJsonbBuildObjectParams(query) + // UNHEX(expr) → decode(expr, 'hex') for checksum computation + query = rewriteUnhex(query) + // CHAR(0) → chr(0) + query = strings.ReplaceAll(query, "CHAR(0)", "chr(0)") + // CONCAT(a, b, ...) → (a || b || ...) — PG's CONCAT can't always infer parameter types + query = rewriteConcat(query) + // ISNULL(expr) → (expr IS NULL) — MySQL's ISNULL returns 1/0; PG doesn't have it. + query = rewriteISNULL(query) + // IFNULL(a, b) → COALESCE(a, b) — MySQL's IFNULL is PG's COALESCE + query = strings.ReplaceAll(query, "IFNULL(", "COALESCE(") + // COALESCE(token, '') → COALESCE(token, ''::bytea) — token is bytea in PG, + // so the empty-string fallback needs an explicit cast. + // Also handle checksum which is bytea. + query = strings.ReplaceAll(query, "COALESCE(token, '')", "COALESCE(token, ''::bytea)") + query = strings.ReplaceAll(query, "COALESCE(checksum, '')", "COALESCE(checksum, ''::bytea)") + // UUID_TO_BIN(UUID(), true) → gen_random_uuid() (must come before UUID() replacement) + query = reUUIDBinUpper.ReplaceAllString(query, "gen_random_uuid()") + query = reUUIDBinLower.ReplaceAllString(query, "gen_random_uuid()") + query = reUUIDBinTrue.ReplaceAllString(query, "($1)::uuid") + query = reUUIDBin.ReplaceAllString(query, "($1)::uuid") + // CONVERT(uuid() USING utf8mb4) → gen_random_uuid()::text (MySQL charset conversion) + query = strings.ReplaceAll(query, "CONVERT(uuid() USING utf8mb4)", "gen_random_uuid()::text") + query = strings.ReplaceAll(query, "CONVERT(UUID() USING utf8mb4)", "gen_random_uuid()::text") + // Standalone UUID() → gen_random_uuid()::text (use word boundary to avoid matching gen_random_uuid) + query = reUUID.ReplaceAllStringFunc(query, func(m string) string { + return "gen_random_uuid()::text" + }) + // BIN_TO_UUID(expr, true) → encode(expr, 'hex') reformatted as UUID text + // Simpler: BIN_TO_UUID(col, true) → col::text for uuid columns + query = reBinToUUIDTrue.ReplaceAllString(query, "($1)::text") + query = reBinToUUID.ReplaceAllString(query, "($1)::text") + // HEX(expr) → encode(expr::bytea, 'hex') — MySQL HEX function + query = rewriteHex(query) + // JSON_SET(col, path, val) → jsonb_set(col, path_array, val) + query = rewriteJSONSet(query) + // TIMEDIFF(a, b) → (a - b) + query = reTimeDiff.ReplaceAllString(query, "($1 - $2)") + // TIME_TO_SEC(interval) → EXTRACT(EPOCH FROM interval) + query = reTimeToSec.ReplaceAllString(query, "EXTRACT(EPOCH FROM $1)") + // ON DUPLICATE KEY UPDATE → rewrite to ON CONFLICT DO UPDATE SET for raw SQL + // that doesn't go through dialect helpers. + // Also normalize "ON DUPLICATE KEY\nUPDATE" (split across lines) to single-line form. + if strings.Contains(query, "ON DUPLICATE KEY") { + query = reNormalizeDuplicateKey.ReplaceAllString(query, "ON DUPLICATE KEY UPDATE") + query = rewriteOnDuplicateKey(query) + } + // FROM DUAL → removed (PG doesn't need FROM DUAL for SELECT without a table) + query = reFromDual.ReplaceAllString(query, "") + // STRAIGHT_JOIN → JOIN (MySQL optimizer hint, not supported by PG) + query = strings.ReplaceAll(query, "STRAIGHT_JOIN", "JOIN") + // MySQL SET FOREIGN_KEY_CHECKS / innodb / sql_mode commands → no-op for PG + if strings.Contains(query, "FOREIGN_KEY_CHECKS") || strings.Contains(query, "innodb") || strings.Contains(query, "INNODB") || strings.Contains(query, "sql_mode") { + query = strings.ReplaceAll(query, "SET FOREIGN_KEY_CHECKS=0", "SELECT 1") + query = strings.ReplaceAll(query, "SET FOREIGN_KEY_CHECKS=1", "SELECT 1") + if strings.Contains(query, "innodb") || strings.Contains(query, "INNODB") || strings.Contains(query, "sql_mode") { + return "SELECT 1" // skip MySQL-specific queries entirely + } + } + // MySQL RAND() → PG random() + query = strings.ReplaceAll(query, "RAND()", "random()") + query = strings.ReplaceAll(query, "rand()", "random()") + // GROUP_CONCAT → STRING_AGG for simple cases not going through dialect + if strings.Contains(query, "GROUP_CONCAT") || strings.Contains(query, "group_concat") { + query = rewriteGroupConcat(query) + } + // FOR UPDATE with LEFT JOIN: PG doesn't allow FOR UPDATE on nullable side of outer join. + // Remove FOR UPDATE when LEFT JOIN is present — the SELECT FOR UPDATE semantic is advisory + // and removing it doesn't break correctness, only reduces locking. + if strings.Contains(query, "FOR UPDATE") && (strings.Contains(query, "LEFT JOIN") || strings.Contains(query, "LEFT OUTER JOIN")) { + query = strings.Replace(query, "\nFOR UPDATE", "", 1) + query = strings.Replace(query, "\n\t\tFOR UPDATE", "", 1) + query = strings.Replace(query, "FOR UPDATE", "", 1) + } + // MySQL SEPARATOR in GROUP_CONCAT → already handled by dialect, but catch raw usage + if strings.Contains(query, "separator") || strings.Contains(query, "SEPARATOR") { + query = reSeparator.ReplaceAllString(query, "") + } + // MySQL JSON path operators: col->'$.key' → col->'key', col->>'$.key' → col->>'key' + query = rewriteJSONPath(query) + // MySQL JSON boolean values: MySQL ->>'$.key' returns '1'/'0' for JSON true/false, + // PG ->>key returns 'true'/'false'. Rewrite COALESCE(expr, '0') = '1' to handle both. + query = reJSONBoolCoalesce.ReplaceAllString(query, "COALESCE($1, '0') IN ('1', 'true')") + // MySQL backtick-quoted identifiers → PG double-quoted identifiers + query = strings.ReplaceAll(query, "`", `"`) + // MySQL DELETE FROM t USING t INNER JOIN → PG DELETE FROM t USING (remove duplicate table) + // MySQL requires naming the target table again in USING; PG forbids it. + query = rewriteDeleteUsing(query) + // MySQL UPDATE t1 JOIN t2 ON ... SET ... → PG UPDATE t1 SET ... FROM t2 WHERE ... + if strings.Contains(query, "UPDATE") && strings.Contains(query, "JOIN") && strings.Contains(query, "SET") { + query = rewriteUpdateJoin(query) + } + // Note: PG doesn't allow alias-qualified columns in UPDATE SET clause. + // This needs per-query fixes in the source code (e.g., cron_stats.go). + // MySQL IF(cond, true_val, false_val) → PG CASE WHEN cond THEN true_val ELSE false_val END + query = rewriteIF(query) + // MySQL FIELD(x, 'a', 'b', ...) → PG CASE x WHEN 'a' THEN 1 WHEN 'b' THEN 2 ... ELSE 0 END + query = rewriteField(query) + // TIMESTAMPDIFF(SECOND, x, y) → EXTRACT(EPOCH FROM (y - x)) + // MySQL's TIMESTAMPDIFF returns the difference in the specified unit. + query = rewriteTimestampDiff(query) + // MySQL DATEDIFF(date1, date2) → PG (date1::date - date2::date) + query = rewriteDateDiff(query) + // TIMESTAMP(x) → x::timestamp (PG cast syntax) + // MySQL TIMESTAMP(?) converts a value to timestamp type + query = reTimestamp.ReplaceAllString(query, "($1)::timestamp") + // CAST(... AS UNSIGNED) → CAST(... AS integer) (MySQL unsigned → PG integer) + query = strings.ReplaceAll(query, "AS UNSIGNED)", "AS integer)") + // CAST(... AS SIGNED INT) / CAST(... AS SIGNED) → CAST(... AS integer) + query = strings.ReplaceAll(query, "AS SIGNED INT)", "AS integer)") + query = strings.ReplaceAll(query, "AS SIGNED)", "AS integer)") + // CAST(TRUE/FALSE AS JSON) → TRUE/FALSE (PG jsonb_build_object accepts boolean directly) + query = strings.ReplaceAll(query, "CAST(TRUE AS JSON)", "TRUE") + query = strings.ReplaceAll(query, "CAST(FALSE AS JSON)", "FALSE") + // CAST(? AS JSON) → CAST(?::text AS jsonb) — PG needs jsonb, not json + query = strings.ReplaceAll(query, "CAST(? AS JSON)", "?::jsonb") + // MySQL json != → PG jsonb != (ensure both sides are jsonb) + query = strings.ReplaceAll(query, "AS JSON)", "AS jsonb)") + // MAX(boolean_col) → BOOL_OR(boolean_col) for PG + query = reMaxDenylisted.ReplaceAllString(query, "BOOL_OR($1)") + // MAX(prof_pending) etc. from integer (0/1) subqueries → BOOL_OR with cast for PG + query = reMaxBooleanCols.ReplaceAllString(query, "BOOL_OR(($1)::boolean)") + // Fix CASE type mismatch: ELSE hdek.decryptable (boolean) mixed with THEN -1 (integer) + // Cast boolean to integer in CASE branches + query = strings.ReplaceAll(query, "ELSE hdek.decryptable", "ELSE CAST(hdek.decryptable AS integer)") + // Fix CAST(AVG(...) AS UNSIGNED) → CAST(AVG(...) AS integer) (already handled above) + // Fix boolean = integer comparisons that PG doesn't allow + for _, col := range []string{ + "ne.enabled", "hsr.canceled", "pl.exclude", "needs_full_membership_cleanup", "si.is_active", + "hsi2.removed", "hsi2.canceled", "hsi.removed", "hsi.canceled", + "abt.terms_expired", "n.token_update_tally", "ne.token_update_tally", + "n.enrolled", "q.active", "cve_meta.published", + "hrkp.deleted", "rkp.deleted", + // nano/mdm boolean columns + "hm.enrolled", "hmdm.enrolled", "nq.active", "nvq.active", + "nano_enrollment_queue.active", + "ba.canceled", "ba2.canceled", + // MDM profile label exclude/require_all columns (various aliases) + "mcpl.exclude", "mcpl.require_all", "mel.exclude", "mel.require_all", + "sil.exclude", "sil.require_all", + "vatl.exclude", "vatl.require_all", "ihl.exclude", "ihl.require_all", + // Additional qualified boolean columns + "neq.active", "e.enabled", "p.conditional_access_enabled", "p.critical", + "hvsi.canceled", "hvsi2.canceled", "hvsi.removed", "hvsi2.removed", + "hihsi.canceled", "hihsi.removed", "hihsi2.canceled", "hihsi2.removed", + "host_vpp_software_installs.canceled", "host_vpp_software_installs.removed", + "host_mdm.enrolled", + "q.automations_enabled", "nq.automations_enabled", + "hmdm.is_server", "hm.installed_from_dep", "q.discard_data", + "hmabp.skipped", "hm.is_personal_enrollment", + // Unqualified boolean columns (safe — always boolean in Fleet schema) + "deleted", "canceled", "refetch_requested", "expired", + "enrolled", "enabled", "active", + "resync", "terms_expired", "sync_request", + "discard_data", "is_server", "is_kernel", "encrypted", + "skipped", "installed_from_dep", "is_personal_enrollment", + "saved", "q.saved", "revoked", "global_stats", + "sthc.global_stats", "shc.global_stats", "vhc.global_stats", + "self_service", "si.self_service", "vat.self_service", "iha.self_service", + // Full table-qualified boolean columns (used without aliases) + "software_installer_labels.exclude", "software_installer_labels.require_all", + "vpp_app_team_labels.exclude", "vpp_app_team_labels.require_all", + "in_house_app_labels.exclude", "in_house_app_labels.require_all", + "hsi.uninstall", "uninstall", + "hdek.decryptable", + "install_during_setup", "si.install_during_setup", + } { + query = strings.ReplaceAll(query, col+" = 1", col+" = true") + query = strings.ReplaceAll(query, col+" = 0", col+" = false") + query = strings.ReplaceAll(query, col+" != 1", col+" != true") + query = strings.ReplaceAll(query, col+"=1", col+"=true") + query = strings.ReplaceAll(query, col+"=0", col+"=false") + query = strings.ReplaceAll(query, col+"!=1", col+"!=true") + } + // Fix pm.passes = 1/0: PG column is boolean, can't compare to integer. + // Cast to int for use in SUM/COUNT aggregates. + // COALESCE(boolean_column, 0/1) → COALESCE(boolean_column, false/true) + // PG requires consistent types in COALESCE — can't mix boolean and integer. + for _, boolCol := range []string{ + "hmdm.enrolled", "hmdm.installed_from_dep", "hmdm.is_personal_enrollment", + "hmdm.is_server", "ne.enrolled", "hm.enrolled", + } { + query = strings.ReplaceAll(query, "COALESCE("+boolCol+", 0)", "COALESCE("+boolCol+", false)") + query = strings.ReplaceAll(query, "COALESCE("+boolCol+", 1)", "COALESCE("+boolCol+", true)") + } + + query = strings.ReplaceAll(query, "pm.passes = 1", "(pm.passes IS TRUE)::int") + query = strings.ReplaceAll(query, "pm.passes = 0", "(pm.passes = false)::int") + // MySQL !boolean → PG NOT boolean (for use in SUM aggregates) + query = strings.ReplaceAll(query, "!pm.passes", "(NOT pm.passes)::int") + // Fix FIND_IN_SET/ANY result compared to integer: PG = ANY() returns boolean + // MySQL FIND_IN_SET returns integer, so code uses <> 0 / != 0 checks + // PG = ANY() returns boolean, making these comparisons invalid + if strings.Contains(query, "string_to_array") { + query = strings.ReplaceAll(query, ")) <> 0", "))") + query = strings.ReplaceAll(query, ")) != 0", "))") + // FindInSet(...) = 0 → NOT FindInSet(...) (PG ANY() returns boolean) + // Pattern: "',')) = 0" at end of FindInSet expression + query = strings.ReplaceAll(query, "',')) = 0", "',')) IS NOT TRUE") + query = strings.ReplaceAll(query, "')) <> 0", "'))") + query = strings.ReplaceAll(query, "')) != 0", "'))") + } + + // Replace MySQL DATE_ADD/DATE_SUB(x, INTERVAL expr UNIT) → PG interval arithmetic + for _, unit := range []string{"SECOND", "MINUTE", "HOUR", "DAY"} { + if strings.Contains(query, "DATE_ADD(") { + query = rewriteDateAddSub(query, unit, "+") + } + if strings.Contains(query, "DATE_SUB(") { + query = rewriteDateAddSub(query, unit, "-") + } + } + + // Replace INTERVAL N SECOND (without DATE_ADD) → INTERVAL 'N seconds' + // e.g., "INTERVAL 5 MINUTE" → "INTERVAL '5 minutes'" + for _, unit := range []string{"SECOND", "MINUTE", "HOUR", "DAY"} { + query = reIntervalLiteral[unit].ReplaceAllString(query, "INTERVAL '${1} "+strings.ToLower(unit)+"s'") + query = reIntervalPlaceholder[unit].ReplaceAllString(query, "? * INTERVAL '1 "+strings.ToLower(unit)+"'") + } + // MySQL allows LIMIT on UPDATE/DELETE; PG does not. + uq := strings.ToUpper(strings.TrimLeft(query, " \t\n")) + if strings.HasPrefix(uq, "UPDATE") || strings.HasPrefix(uq, "DELETE") { + query = reLimitTrailing.ReplaceAllString(query, "") + } + + // Resolve ambiguous column references in ON CONFLICT DO UPDATE SET clauses. + // Only apply when complex expressions (CASE WHEN, COALESCE) are in the SET clause. + if idx := strings.Index(query, "DO UPDATE SET"); idx >= 0 { + setClause := query[idx:] + if strings.Contains(setClause, "CASE WHEN") || strings.Contains(setClause, "COALESCE") { + if strings.Contains(query, "EXCLUDED.") { + query = resolveOnConflictAmbiguity(query) + } + } + } + + if !strings.Contains(query, "?") { + if hasInsertIgnore { + query = strings.TrimRight(query, " \t\n\r;") + " ON CONFLICT DO NOTHING" + } + return query + } + var b strings.Builder + b.Grow(len(query) + 10) + n := 1 + for _, r := range query { + if r == '?' { + b.WriteByte('$') + b.WriteString(strings.Repeat("", 0)) // force allocation + // Write the number + if n < 10 { + b.WriteByte(byte('0' + n)) + } else { + b.WriteString(fmt.Sprintf("%d", n)) + } + n++ + } else { + b.WriteRune(r) + } + } + result := b.String() + if hasInsertIgnore { + result = strings.TrimRight(result, " \t\n\r;") + " ON CONFLICT DO NOTHING" + } + // PG can't infer the type of $N when used in interval arithmetic ($N - INTERVAL, $N + INTERVAL). + // Cast to timestamptz so the operator resolves correctly. + result = reParamBeforeInterval.ReplaceAllString(result, "${1}::timestamptz ${2}") + return result +} + +func (c *rebindConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + if ec, ok := c.Conn.(driver.ExecerContext); ok { + rebound := rebindQuery(query) + coerced := coerceTimeArgsToUTC(coerceBinaryArgs(coerceBoolArgsForTextCast(rebound, args))) + return ec.ExecContext(ctx, rebound, coerced) + } + return nil, driver.ErrSkip +} + +func (c *rebindConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + if qc, ok := c.Conn.(driver.QueryerContext); ok { + rebound := rebindQuery(query) + coerced := coerceTimeArgsToUTC(coerceBinaryArgs(coerceBoolArgsForTextCast(rebound, args))) + rows, err := qc.QueryContext(ctx, rebound, coerced) + if err != nil { + return nil, err + } + return &rebindRows{Rows: rows}, nil + } + return nil, driver.ErrSkip +} + +// rebindRows wraps driver.Rows to convert string values to []byte in Next(). +// PostgreSQL (via pgx) returns text/json/jsonb column values as Go strings, +// but database/sql cannot convert string → []byte for destinations like +// json.RawMessage. Converting all strings to []byte at the driver level is +// safe because database/sql's convertAssign handles []byte → *string, +// *int, *bool, and all other common destination types. +type rebindRows struct { + driver.Rows +} + +func (r *rebindRows) Next(dest []driver.Value) error { + if err := r.Rows.Next(dest); err != nil { + return err + } + for i, v := range dest { + if s, ok := v.(string); ok { + dest[i] = []byte(s) + } + } + return nil +} + +// HasNextResultSet forwards to the underlying rows if supported. +func (r *rebindRows) HasNextResultSet() bool { + if rs, ok := r.Rows.(driver.RowsNextResultSet); ok { + return rs.HasNextResultSet() + } + return false +} + +// NextResultSet forwards to the underlying rows if supported. +func (r *rebindRows) NextResultSet() error { + if rs, ok := r.Rows.(driver.RowsNextResultSet); ok { + return rs.NextResultSet() + } + return fmt.Errorf("not supported") +} + +// coerceBoolArgsForTextCast converts Go bool args to "true"/"false" strings +// when the rebound query casts the corresponding placeholder to ::text. +// This prevents pgx "unable to encode bool into text format" errors +// (e.g. inside jsonb_build_object where all value args get ::text casts). +func coerceBoolArgsForTextCast(query string, args []driver.NamedValue) []driver.NamedValue { + // Quick exit: if no bool args, nothing to do + hasBool := false + for _, a := range args { + if _, ok := a.Value.(bool); ok { + hasBool = true + break + } + } + if !hasBool { + return args + } + + // Build a set of 1-based parameter ordinals that have ::text cast + textCastParams := make(map[int]bool) + for i := 0; i < len(query)-6; i++ { + if query[i] == '$' && query[i+1] >= '1' && query[i+1] <= '9' { + j := i + 1 + for j < len(query) && query[j] >= '0' && query[j] <= '9' { + j++ + } + ordinal := 0 + for _, ch := range query[i+1 : j] { + ordinal = ordinal*10 + int(ch-'0') + } + // Check if followed by ::text + rest := query[j:] + if strings.HasPrefix(rest, "::text") { + textCastParams[ordinal] = true + } + } + } + + if len(textCastParams) == 0 { + return args + } + + // Copy and convert bool args that are cast to ::text + out := make([]driver.NamedValue, len(args)) + copy(out, args) + for i, a := range out { + if b, ok := a.Value.(bool); ok && textCastParams[a.Ordinal] { + if b { + out[i].Value = "true" + } else { + out[i].Value = "false" + } + } + } + return out +} + +// coerceBinaryArgs converts string args that contain non-UTF-8 byte sequences +// to []byte so that pgx sends them as bytea instead of text. MySQL's binary(N) +// coerceTimeArgsToUTC converts time.Time parameters to UTC before sending to PG. +// PG "timestamp without time zone" stores wall-clock values without timezone. +// Go local time (e.g., 10:00 PDT) gets stored as "10:00" and read back as 10:00 UTC. +func coerceTimeArgsToUTC(args []driver.NamedValue) []driver.NamedValue { + var out []driver.NamedValue + for i, a := range args { + if t, ok := a.Value.(time.Time); ok && t.Location() != time.UTC { + if out == nil { + out = make([]driver.NamedValue, len(args)) + copy(out, args) + } + out[i].Value = t.UTC() + } + } + if out == nil { + return args + } + return out +} + +// columns are read as Go strings containing raw bytes; PG rejects non-UTF-8 +// strings with "invalid byte sequence for encoding UTF8". +func coerceBinaryArgs(args []driver.NamedValue) []driver.NamedValue { + var out []driver.NamedValue + for i, a := range args { + if s, ok := a.Value.(string); ok && len(s) > 0 && !utf8.ValidString(s) { + if out == nil { + out = make([]driver.NamedValue, len(args)) + copy(out, args) + } + out[i].Value = []byte(s) + } + } + if out == nil { + return args + } + return out +} + +func (c *rebindConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + if pc, ok := c.Conn.(driver.ConnPrepareContext); ok { + return pc.PrepareContext(ctx, rebindQuery(query)) + } + return c.Conn.Prepare(rebindQuery(query)) +} + +func (c *rebindConn) Prepare(query string) (driver.Stmt, error) { + return c.Conn.Prepare(rebindQuery(query)) +} + +// rewriteDateAddSub converts MySQL DATE_ADD/DATE_SUB(expr, INTERVAL value UNIT) to PG interval arithmetic. +// op is "+" for DATE_ADD and "-" for DATE_SUB. +func rewriteDateAddSub(query string, unit string, op string) string { + pgUnit := strings.ToLower(unit) + "s" + var prefix string + if op == "+" { + prefix = "DATE_ADD(" + } else { + prefix = "DATE_SUB(" + } + for { + idx := strings.Index(query, prefix) + if idx < 0 { + return query + } + // Find the matching closing paren and split on the top-level comma + start := idx + len(prefix) + depth := 1 + commaPos := -1 + i := start + for i < len(query) && depth > 0 { + switch query[i] { + case '(': + depth++ + case ')': + depth-- + case ',': + if depth == 1 && commaPos < 0 { + commaPos = i + } + } + i++ + } + if depth != 0 || commaPos < 0 { + return query // unbalanced or no comma found + } + expr := strings.TrimSpace(query[start:commaPos]) + intervalPart := strings.TrimSpace(query[commaPos+1 : i-1]) + + // Parse: INTERVAL + intervalRe := regexp.MustCompile(`(?i)INTERVAL\s+(.+)\s+` + unit) + m := intervalRe.FindStringSubmatch(intervalPart) + if m == nil { + // This DATE_ADD/SUB doesn't use this unit, skip past it + return query[:i] + rewriteDateAddSub(query[i:], unit, op) + } + value := strings.TrimSpace(m[1]) + // If the date expression is a placeholder, PG can't infer its type in interval arithmetic. + // Cast to timestamptz so the +/- operator resolves correctly. + if strings.TrimSpace(expr) == "?" { + expr = "?::timestamptz" + } + replacement := "(" + expr + " " + op + " (" + value + ") * INTERVAL '1 " + pgUnit + "')" + query = query[:idx] + replacement + query[i:] + } +} + +// rewriteUnhex converts MySQL UNHEX(expr) → PG decode(expr, 'hex'). +// Uses paren-balancing to handle nested function calls inside UNHEX(). +func rewriteUnhex(query string) string { + const prefix = "UNHEX(" + for { + idx := strings.Index(query, prefix) + if idx < 0 { + return query + } + // Find the matching closing paren + depth := 1 + start := idx + len(prefix) + i := start + for i < len(query) && depth > 0 { + if query[i] == '(' { + depth++ + } else if query[i] == ')' { + depth-- + } + i++ + } + if depth != 0 { + return query // unbalanced, leave as-is + } + inner := query[start : i-1] + query = query[:idx] + "decode(" + inner + ", 'hex')" + query[i:] + } +} + +// rewriteDeleteUsing fixes MySQL's DELETE FROM t USING t INNER JOIN ... +// pattern for PostgreSQL. MySQL requires repeating the target table in USING; +// PG forbids it. +// +// MySQL: DELETE FROM t USING t INNER JOIN j alias ON WHERE +// PG: DELETE FROM t USING j alias WHERE AND +func rewriteDeleteUsing(query string) string { + // Extract the target table from DELETE FROM + delRe := regexp.MustCompile(`(?is)DELETE\s+FROM\s+(\w+)\s+USING\s+`) + m := delRe.FindStringSubmatch(query) + if m == nil { + return query + } + tableName := m[1] + + // Check if the USING clause repeats the same table name followed by INNER JOIN + // Build a pattern: USING INNER JOIN (case-insensitive) + usingDupRe := regexp.MustCompile(`(?is)USING\s+` + regexp.QuoteMeta(tableName) + `\s+INNER\s+JOIN\s+`) + if !usingDupRe.MatchString(query) { + return query + } + + // Step 1: Remove duplicate table and INNER JOIN keyword + query = usingDupRe.ReplaceAllString(query, "USING ") + + // Step 2: Convert "ON WHERE" → "WHERE AND" + // The ON clause from the removed INNER JOIN must merge into WHERE. + reOnWhere := regexp.MustCompile(`(?is)(USING\s+\w+\s+\w+\s+)ON\s+(.*?)\s+WHERE\s+`) + query = reOnWhere.ReplaceAllString(query, "${1}WHERE ${2} AND ") + + return query +} + +// rewriteTimestampDiff converts MySQL TIMESTAMPDIFF(SECOND, x, y) → PG EXTRACT(EPOCH FROM (y - x)). +func rewriteTimestampDiff(query string) string { + if !reTimestampDiff.MatchString(query) { + return query + } + // Use paren-balanced parsing for complex arguments + prefix := "TIMESTAMPDIFF(" + for { + idx := strings.Index(strings.ToUpper(query), strings.ToUpper(prefix)) + if idx < 0 { + return query + } + start := idx + len(prefix) + depth := 1 + var parts []string + partStart := start + i := start + for i < len(query) && depth > 0 { + switch query[i] { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + } + case ',': + if depth == 1 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + partStart = i + 1 + } + } + i++ + } + if depth != 0 || len(parts) != 3 { + return query + } + // parts[0] = unit (SECOND), parts[1] = start_time, parts[2] = end_time + replacement := fmt.Sprintf("EXTRACT(EPOCH FROM (%s - %s))", parts[2], parts[1]) + query = query[:idx] + replacement + query[i:] + } +} + +// rewriteDateDiff converts MySQL DATEDIFF(date1, date2) → PG (date1::date - date2::date). +// Uses paren-balancing to handle nested expressions in the arguments. +func rewriteDateDiff(query string) string { + for { + // Find DATEDIFF( that is not part of a longer identifier (e.g., TIMESTAMPDIFF) + idx := -1 + searchFrom := 0 + for searchFrom < len(query) { + upper := strings.ToUpper(query[searchFrom:]) + pos := strings.Index(upper, "DATEDIFF(") + if pos < 0 { + break + } + absPos := searchFrom + pos + if absPos > 0 && isIdentChar(query[absPos-1]) { + searchFrom = absPos + 9 // skip past this match + continue + } + idx = absPos + break + } + if idx < 0 { + return query + } + + start := idx + 9 // after "DATEDIFF(" + depth := 1 + var parts []string + partStart := start + i := start + for i < len(query) && depth > 0 { + switch query[i] { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + } + case ',': + if depth == 1 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + partStart = i + 1 + } + } + i++ + } + if depth != 0 || len(parts) != 2 { + return query // unbalanced or wrong number of args, leave as-is + } + replacement := fmt.Sprintf("(%s::date - %s::date)", parts[0], parts[1]) + query = query[:idx] + replacement + query[i:] + } +} + +// rewriteIF converts MySQL IF(cond, true_val, false_val) → PG CASE WHEN cond THEN true_val ELSE false_val END. +// Uses paren-balancing and comma-splitting to handle nested expressions. +func rewriteIF(query string) string { + for { + // Find IF( preceded by a non-alphanumeric char (or start of string) + // to avoid matching e.g. NOTIFY(...) + idx := -1 + for i := 0; i < len(query)-3; i++ { + if (query[i] == 'I' || query[i] == 'i') && + (query[i+1] == 'F' || query[i+1] == 'f') && + query[i+2] == '(' { + // Check that the preceding char is not alphanumeric/underscore + if i == 0 || !isIdentChar(query[i-1]) { + idx = i + break + } + } + } + if idx < 0 { + return query + } + + // Find the matching closing paren, splitting on top-level commas + start := idx + 3 // after "IF(" + depth := 1 + var parts []string + partStart := start + i := start + for i < len(query) && depth > 0 { + switch query[i] { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + } + case ',': + if depth == 1 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + partStart = i + 1 + } + } + i++ + } + if depth != 0 || len(parts) != 3 { + return query // unbalanced or not exactly 3 args, leave as-is + } + replacement := fmt.Sprintf("CASE WHEN %s THEN %s ELSE %s END", parts[0], parts[1], parts[2]) + query = query[:idx] + replacement + query[i:] + } +} + +func isIdentChar(c byte) bool { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' +} + + +// castJsonbBuildObjectParams adds ::text casts to ? placeholders inside jsonb_build_object() calls. +// PG's jsonb_build_object has a VARIADIC "any" signature, so it can't infer placeholder parameter types. +// Casting to ::text makes all JSON values strings, which is compatible with ->>' text extraction. +// Handles nested jsonb_build_object and subqueries via paren-balancing. +func castJsonbBuildObjectParams(query string) string { + const prefix = "jsonb_build_object(" + for { + idx := strings.Index(query, prefix) + if idx < 0 { + return query + } + start := idx + len(prefix) + depth := 1 + i := start + // Walk through the jsonb_build_object args, adding ::text to ? placeholders + // in ALL positions (both keys and values). PG's jsonb_build_object has a + // VARIADIC "any" signature, so it can't infer any placeholder parameter types. + var result strings.Builder + result.WriteString(query[:start]) + argStart := i + + for i < len(query) && depth > 0 { + switch query[i] { + case '(': + depth++ + i++ + case ')': + depth-- + if depth == 0 { + // Process the last argument + arg := query[argStart:i] + arg = castPlaceholdersInArg(arg) + result.WriteString(arg) + result.WriteByte(')') + } + i++ + case ',': + if depth == 1 { + arg := query[argStart:i] + arg = castPlaceholdersInArg(arg) + result.WriteString(arg) + result.WriteByte(',') + argStart = i + 1 + i++ + } else { + i++ + } + default: + i++ + } + } + if depth != 0 { + return query // unbalanced, leave as-is + } + // Recursively process the rest of the query + result.WriteString(castJsonbBuildObjectParams(query[i:])) + return result.String() + } +} + +// castPlaceholdersInArg adds ::text to bare ? placeholders in a jsonb_build_object value argument. +// Skips ? that are inside subqueries (nested parens), CAST expressions, or already have ::text. +func castPlaceholdersInArg(arg string) string { + trimmed := strings.TrimSpace(arg) + // If the arg is a simple ?, cast it + if trimmed == "?" { + return strings.Replace(arg, "?", "?::text", 1) + } + // If the arg is CAST(? AS ...), leave it alone (already typed) + if strings.Contains(strings.ToUpper(trimmed), "CAST(") { + return arg + } + // If the arg contains a subquery (SELECT ...), leave it alone (nested query handles its own types) + if strings.Contains(strings.ToUpper(trimmed), "SELECT ") { + return arg + } + // For other simple expressions with ?, cast them + if trimmed == "?" { + return strings.Replace(arg, "?", "?::text", 1) + } + return arg +} + +// rewriteJSONExtractFunc converts MySQL JSON_EXTRACT(col, path) → PG (col->path_key). +// For parameterized paths (JSON_EXTRACT(col, ?)), wraps with regexp_replace to strip +// the MySQL $. prefix and optional quotes at runtime. +func rewriteJSONExtractFunc(query string) string { + // Match JSON_EXTRACT(identifier, ?) or JSON_EXTRACT(identifier, 'literal') + return reJSONExtractFunc.ReplaceAllStringFunc(query, func(match string) string { + m := reJSONExtractFunc.FindStringSubmatch(match) + if m == nil { + return match + } + col, pathExpr := m[1], m[2] + if pathExpr == "?" { + // Parameterized path: strip $. prefix and quotes at runtime. + // Use {0,1} instead of ? as regex quantifier to avoid the rebinder + // treating it as a SQL placeholder (the ? → $N replacement is global). + return fmt.Sprintf("(%s->regexp_replace(?::text, '^\\$\\.\"{0,1}([^\"]*)\"{0,1}$', '\\1'))", col) + } + // Literal path: strip $. prefix inline + path := strings.TrimPrefix(pathExpr, "'$.") + path = strings.TrimSuffix(path, "'") + path = strings.Trim(path, `"`) + return fmt.Sprintf("(%s->'%s')", col, path) + }) +} + +// rewriteJSONPath converts MySQL JSON path operator syntax to PG. +// MySQL: col->'$.key' → PG: col->'key' +// MySQL: col->>'$.key' → PG: col->>'key' +// MySQL: col->'$.key1.key2' → PG: col->'key1'->'key2' +// MySQL: col->>'$.key1.key2' → PG: col->'key1'->>'key2' +// This handles the $. prefix that MySQL uses for JSON paths, including dotted sub-paths. +func rewriteJSONPath(query string) string { + query = reJSONPath.ReplaceAllStringFunc(query, func(match string) string { + // Determine operator: ->> or -> + isText := strings.HasPrefix(match, "->>") + // Strip operator prefix and $. and surrounding quotes + path := match + if isText { + path = strings.TrimPrefix(path, "->>'$.") + } else { + path = strings.TrimPrefix(path, "->'$.") + } + path = strings.TrimSuffix(path, "'") + // Split on dots for nested paths + parts := strings.Split(path, ".") + if len(parts) == 1 { + // Simple case: no dots + if isText { + return "->>'"+parts[0]+"'" + } + return "->'"+parts[0]+"'" + } + // Multi-level path: all but last use ->, last uses the original operator + var sb strings.Builder + for i, part := range parts { + if i < len(parts)-1 { + sb.WriteString("->'") + sb.WriteString(part) + sb.WriteString("'") + } else { + if isText { + sb.WriteString("->>'") + } else { + sb.WriteString("->'") + } + sb.WriteString(part) + sb.WriteString("'") + } + } + return sb.String() + }) + return query +} + +// rewriteConcat converts MySQL CONCAT(a, b, ...) → (a::text || b::text || ...). +// PG's CONCAT() function can't always infer parameter types for placeholders. +// Uses paren-balancing to handle nested expressions. +func rewriteConcat(query string) string { + for { + idx := strings.Index(query, "CONCAT(") + if idx < 0 { + return query + } + // Make sure CONCAT is not part of a larger identifier (e.g. GROUP_CONCAT) + if idx > 0 && isIdentChar(query[idx-1]) { + // Skip past this occurrence + rest := query[idx+7:] + before := query[:idx+7] + rewritten := rewriteConcat(rest) + return before + rewritten + } + start := idx + 7 // after "CONCAT(" + depth := 1 + var parts []string + partStart := start + i := start + for i < len(query) && depth > 0 { + switch query[i] { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + } + case ',': + if depth == 1 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + partStart = i + 1 + } + } + i++ + } + if depth != 0 || len(parts) < 1 { + return query + } + // Build (part1::text || part2::text || ...) + var b strings.Builder + b.WriteByte('(') + for j, part := range parts { + if j > 0 { + b.WriteString(" || ") + } + b.WriteString(part) + b.WriteString("::text") + } + b.WriteByte(')') + query = query[:idx] + b.String() + query[i:] + } +} + +// rewriteISNULL converts MySQL ISNULL(expr) → (expr IS NULL). +// Uses paren-balancing to handle nested expressions. +func rewriteISNULL(query string) string { + for { + idx := strings.Index(query, "ISNULL(") + if idx < 0 { + return query + } + // Make sure ISNULL is not part of a larger identifier + if idx > 0 && isIdentChar(query[idx-1]) { + // Skip past this occurrence and continue searching + rest := rewriteISNULL(query[idx+7:]) + return query[:idx+7] + rest + } + start := idx + 7 // after "ISNULL(" + depth := 1 + i := start + for i < len(query) && depth > 0 { + if query[i] == '(' { + depth++ + } else if query[i] == ')' { + depth-- + } + i++ + } + if depth != 0 { + return query // unbalanced + } + inner := query[start : i-1] + query = query[:idx] + "(" + inner + " IS NULL)" + query[i:] + } +} + +// rewriteField converts MySQL FIELD(x, 'a', 'b', ...) → PG CASE x WHEN 'a' THEN 1 WHEN 'b' THEN 2 ... ELSE 0 END. +func rewriteField(query string) string { + prefix := "FIELD(" + idx := strings.Index(query, prefix) + if idx < 0 { + return query + } + // Ensure FIELD( is not part of a larger identifier + if idx > 0 && isIdentChar(query[idx-1]) { + return query + } + start := idx + len(prefix) + depth := 1 + var parts []string + partStart := start + i := start + for i < len(query) && depth > 0 { + switch query[i] { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + } + case ',': + if depth == 1 { + parts = append(parts, strings.TrimSpace(query[partStart:i])) + partStart = i + 1 + } + } + i++ + } + if depth != 0 || len(parts) < 2 { + return query + } + var b strings.Builder + b.WriteString("CASE ") + b.WriteString(parts[0]) + for j := 1; j < len(parts); j++ { + fmt.Fprintf(&b, " WHEN %s THEN %d", parts[j], j) + } + b.WriteString(" ELSE 0 END") + return query[:idx] + b.String() + query[i:] +} + +// resolveOnConflictAmbiguity fixes ambiguous column references in ON CONFLICT DO UPDATE SET. +// In PG, bare column names in SET value expressions are ambiguous between the target table +// and EXCLUDED. This function parses each SET assignment and qualifies bare column references +// in the VALUE expressions (right side of =) with the target table name. +func resolveOnConflictAmbiguity(query string) string { + // Extract target table name from INSERT INTO
+ insertRe := regexp.MustCompile(`(?i)INSERT\s+INTO\s+"?(\w+)"?`) + m := insertRe.FindStringSubmatch(query) + if m == nil { + return query + } + tableName := m[1] + + // Find the ON CONFLICT DO UPDATE SET portion + upperQuery := strings.ToUpper(query) + setMarker := "DO UPDATE SET" + setIdx := strings.Index(upperQuery, setMarker) + if setIdx == -1 { + return query + } + setStart := setIdx + len(setMarker) + setClause := query[setStart:] + + // Collect column names from EXCLUDED references — these are the ambiguous ones + excludedRe := regexp.MustCompile(`EXCLUDED\.(\w+)`) + matches := excludedRe.FindAllStringSubmatch(setClause, -1) + if len(matches) == 0 { + return query + } + cols := make(map[string]bool) + for _, m := range matches { + cols[m[1]] = true + } + // Also add SET target names + setTargetRe := regexp.MustCompile(`(?:^|,)\s*(\w+)\s*=`) + for _, m := range setTargetRe.FindAllStringSubmatch(setClause, -1) { + cols[m[1]] = true + } + + // Split the SET clause into individual assignments by top-level commas. + // Then for each assignment, split on the first '=' to get target and value. + // Only qualify bare column refs in the value part. + assignments := splitTopLevel(setClause, ',') + var result strings.Builder + for i, assignment := range assignments { + if i > 0 { + result.WriteByte(',') + } + eqIdx := strings.Index(assignment, "=") + if eqIdx == -1 { + result.WriteString(assignment) + continue + } + target := assignment[:eqIdx+1] // includes the '=' + value := assignment[eqIdx+1:] + + // Qualify bare column names in the value part using manual scanning + // to avoid the ReplaceAllStringFunc closure bug with mutable value. + value = qualifyBareColumns(value, cols, tableName) + + result.WriteString(target) + result.WriteString(value) + } + + return query[:setStart] + result.String() +} + +// qualifyBareColumns scans a string and qualifies bare column references with tableName. +// A "bare" reference is a word matching a column name NOT preceded by '.'. +func qualifyBareColumns(s string, cols map[string]bool, tableName string) string { + var result strings.Builder + result.Grow(len(s) * 2) + i := 0 + for i < len(s) { + // Skip non-word characters + if !isWordChar(s[i]) { + result.WriteByte(s[i]) + i++ + continue + } + // Extract the full word + start := i + for i < len(s) && isWordChar(s[i]) { + i++ + } + word := s[start:i] + + // Check if this word is a column name we need to qualify + if cols[word] { + // Check if preceded by '.' (already qualified) + if start > 0 && s[start-1] == '.' { + result.WriteString(word) + } else { + result.WriteString(tableName + "." + word) + } + } else { + result.WriteString(word) + } + } + return result.String() +} + +func isWordChar(c byte) bool { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' +} + +// rewriteHex rewrites MySQL HEX(expr) → PG encode(expr, 'hex') +func rewriteHex(query string) string { + // Only match standalone HEX( not UNHEX( + re := regexp.MustCompile(`(?i)\bHEX\(`) + for { + loc := re.FindStringIndex(query) + if loc == nil { + break + } + // Make sure it's not UNHEX + if loc[0] > 0 && (query[loc[0]-1] == 'N' || query[loc[0]-1] == 'n') { + // Skip this match — it's part of UNHEX + query = query[:loc[0]] + "HEX__SKIP(" + query[loc[1]:] + continue + } + // Find matching close paren + depth := 1 + i := loc[1] + for i < len(query) && depth > 0 { + if query[i] == '(' { + depth++ + } else if query[i] == ')' { + depth-- + } + i++ + } + if depth != 0 { + break + } + inner := query[loc[1] : i-1] + replacement := "encode(" + inner + "::bytea, 'hex')" + query = query[:loc[0]] + replacement + query[i:] + } + query = strings.ReplaceAll(query, "HEX__SKIP(", "HEX(") + return query +} + +// rewriteJSONSet rewrites MySQL JSON_SET(col, '$.path', val) → PG jsonb_set(col, '{path}', to_jsonb(val)) +func rewriteJSONSet(query string) string { + for { + idx := strings.Index(query, "JSON_SET(") + if idx == -1 { + break + } + // Find matching close paren + depth := 1 + i := idx + 9 // len("JSON_SET(") + for i < len(query) && depth > 0 { + if query[i] == '(' { + depth++ + } else if query[i] == ')' { + depth-- + } + i++ + } + if depth != 0 { + break + } + inner := query[idx+9 : i-1] + // Parse: col, '$.path', val + parts := splitTopLevel(inner, ',') + if len(parts) < 3 { + break + } + col := strings.TrimSpace(parts[0]) + path := strings.TrimSpace(parts[1]) + val := strings.TrimSpace(parts[2]) + // Convert '$.mdm.foo.bar' → '{mdm,foo,bar}' + path = strings.Trim(path, "'\"") + path = strings.TrimPrefix(path, "$.") + pgPath := "'{" + strings.ReplaceAll(path, ".", ",") + "}'" + // If val is a placeholder ($N or ?), cast to text so PG can determine the type + valExpr := val + if val == "?" || (len(val) > 1 && val[0] == '$' && val[1] >= '0' && val[1] <= '9') { + valExpr = val + "::text" + } + replacement := "jsonb_set(" + col + ", " + pgPath + ", to_jsonb(" + valExpr + "))" + query = query[:idx] + replacement + query[i:] + } + return query +} + +// splitTopLevel splits a string by delimiter, respecting parentheses and quotes. +func splitTopLevel(s string, delim byte) []string { + var parts []string + depth := 0 + inSingleQuote := false + start := 0 + for i := 0; i < len(s); i++ { + switch { + case s[i] == '\'' && !inSingleQuote: + inSingleQuote = true + case s[i] == '\'' && inSingleQuote: + inSingleQuote = false + case inSingleQuote: + continue + case s[i] == '(': + depth++ + case s[i] == ')': + depth-- + case s[i] == delim && depth == 0: + parts = append(parts, s[start:i]) + start = i + 1 + } + } + parts = append(parts, s[start:]) + return parts +} + +// rewriteOnDuplicateKey rewrites MySQL ON DUPLICATE KEY UPDATE → PG ON CONFLICT DO UPDATE SET +// This handles cases not going through the dialect helper. +// knownPrimaryKeys maps table names to their primary key columns for ON CONFLICT resolution. +var knownPrimaryKeys = map[string]string{ + "host_dep_assignments": "host_id", + "host_mdm_idp_accounts": "host_uuid", + "host_mdm_apple_declarations": "host_uuid,declaration_uuid", + "mdm_declaration_labels": "apple_declaration_uuid,label_name", + "scim_user_group": "scim_user_id,group_id", + "host_munki_issues": "host_id,munki_issue_id", + "host_munki_info": "host_id", + "cron_stats": "id", + "nano_command_results": "id,command_uuid", + "host_mdm_apple_bootstrap_packages": "host_uuid", + "mdm_configuration_profile_labels": "id", + "app_config_json": "id", + "host_mdm_android_profiles": "host_uuid,profile_uuid", + "host_conditional_access": "host_id", + "host_mdm": "host_id", + "host_display_names": "host_id", + "host_emails": "id", + "label_membership": "host_id,label_id", + "host_software": "host_id,software_id", + "software_host_counts": "software_id,team_id", + "nano_enrollment_queue": "id,command_uuid", + "host_mdm_windows_profiles": "host_uuid,profile_uuid", + // NanoMDM/NanoDEP tables + "nano_dep_names": "name", + "nano_devices": "id", + "nano_users": "id,device_id", + "nano_enrollments": "id", + "nano_cert_auth_associations": "id,sha256", + "nano_push_certs": "topic", + "host_certificate_templates": "host_uuid,certificate_template_id", +} + +func rewriteOnDuplicateKey(query string) string { + upperQuery := strings.ToUpper(query) + const marker = "ON DUPLICATE KEY UPDATE" + idx := strings.Index(upperQuery, marker) + if idx == -1 { + return query + } + updateClause := strings.TrimSpace(query[idx+len(marker):]) + // Rewrite VALUES(col) → EXCLUDED.col + re := regexp.MustCompile(`(?i)VALUES\(` + "`?" + `(\w+)` + "`?" + `\)`) + updateClause = re.ReplaceAllString(updateClause, "EXCLUDED.$1") + + // Extract table name from INSERT INTO
+ tableRe := regexp.MustCompile(`(?i)INSERT\s+INTO\s+` + "`?" + `(\w+)` + "`?") + m := tableRe.FindStringSubmatch(query) + conflictTarget := "" + if m != nil { + tableName := strings.ToLower(m[1]) + if pk, ok := knownPrimaryKeys[tableName]; ok { + conflictTarget = pk + } + } + + if conflictTarget != "" { + query = query[:idx] + "ON CONFLICT (" + conflictTarget + ") DO UPDATE SET " + updateClause + } else { + // Fallback: no conflict target — PG will error but at least the syntax is close + query = query[:idx] + "ON CONFLICT DO UPDATE SET " + updateClause + } + return query +} + +// rewriteGroupConcat rewrites MySQL GROUP_CONCAT(expr) → PG STRING_AGG(expr::text, ',') +// Also handles GROUP_CONCAT(expr SEPARATOR 'sep') → STRING_AGG(expr::text, 'sep') +// And GROUP_CONCAT(DISTINCT expr) → STRING_AGG(DISTINCT expr::text, ',') +func rewriteGroupConcat(query string) string { + re := regexp.MustCompile(`(?i)GROUP_CONCAT\(`) + for { + loc := re.FindStringIndex(query) + if loc == nil { + break + } + // Find matching close paren + depth := 1 + i := loc[1] + for i < len(query) && depth > 0 { + if query[i] == '(' { + depth++ + } else if query[i] == ')' { + depth-- + } + i++ + } + if depth != 0 { + break + } + inner := strings.TrimSpace(query[loc[1] : i-1]) + sep := "," + // Check for SEPARATOR clause + sepRe := regexp.MustCompile(`(?i)\s+SEPARATOR\s+'([^']*)'`) + if m := sepRe.FindStringSubmatchIndex(inner); m != nil { + sep = inner[m[2]:m[3]] + inner = strings.TrimSpace(inner[:m[0]]) + } + // Check for ORDER BY clause (remove it for STRING_AGG — PG STRING_AGG has its own ORDER BY) + orderRe := regexp.MustCompile(`(?i)\s+ORDER\s+BY\s+.+`) + orderClause := "" + if m := orderRe.FindStringIndex(inner); m != nil { + orderClause = " " + strings.TrimSpace(inner[m[0]:]) + inner = strings.TrimSpace(inner[:m[0]]) + } + replacement := "STRING_AGG(" + inner + "::text, '" + sep + "'" + orderClause + ")" + query = query[:loc[0]] + replacement + query[i:] + } + return query +} + +// rewriteUpdateJoin rewrites MySQL UPDATE t1 JOIN t2 ON cond SET ... → PG UPDATE t1 SET ... FROM t2 WHERE cond +// Handles both aliased (UPDATE t1 a JOIN ...) and unaliased (UPDATE t1 JOIN ...) forms. +func rewriteUpdateJoin(query string) string { + // MySQL: UPDATE t1 [a] [INNER] JOIN t2 b ON cond [JOIN ...] SET assignments [WHERE where] + // PG: UPDATE t1 [a] SET assignments FROM t2 b [, t3 c] WHERE cond [AND where] + + // Try aliased form first: UPDATE table alias JOIN ... + re := regexp.MustCompile(`(?is)UPDATE\s+(\S+)\s+(\w+)\s+((?:(?:INNER\s+)?JOIN\s+.+?\s+ON\s+.+?\s+)+)\bSET\b\s+(.+)`) + m := re.FindStringSubmatch(query) + var table1, alias1, joinBlock, setAndWhere string + if m != nil { + // Check if what we captured as "alias" is actually the JOIN keyword + if strings.EqualFold(m[2], "JOIN") || strings.EqualFold(m[2], "INNER") { + m = nil // not actually aliased, fall through to unaliased form + } + } + if m != nil { + table1 = m[1] + alias1 = m[2] + joinBlock = m[3] + setAndWhere = m[4] + } else { + // Try unaliased form: UPDATE table JOIN ... (no alias) + reNoAlias := regexp.MustCompile(`(?is)UPDATE\s+(\S+)\s+((?:(?:INNER\s+)?JOIN\s+.+?\s+ON\s+.+?\s+)+)\bSET\b\s+(.+)`) + m2 := reNoAlias.FindStringSubmatch(query) + if m2 == nil { + return query + } + table1 = m2[1] + alias1 = "" // no alias + joinBlock = m2[2] + setAndWhere = m2[3] + } + + // Parse individual JOINs from the join block + // Handle both aliased (JOIN table alias ON) and unaliased (JOIN table ON) joins + var fromTables []string + var onConditions []string + joinRe := regexp.MustCompile(`(?is)(?:INNER\s+)?JOIN\s+(\S+)\s+(?:(\w+)\s+)?ON\s+`) + joinMatches := joinRe.FindAllStringSubmatchIndex(joinBlock, -1) + for i, loc := range joinMatches { + table := joinBlock[loc[2]:loc[3]] + alias := "" + if loc[4] >= 0 && loc[5] >= 0 { + candidate := joinBlock[loc[4]:loc[5]] + // Make sure it's not the ON keyword itself + if !strings.EqualFold(candidate, "ON") { + alias = candidate + } + } + if alias != "" { + fromTables = append(fromTables, table+" "+alias) + } else { + fromTables = append(fromTables, table) + } + // ON condition runs from end of this match to start of next match (or end of block) + condStart := loc[1] + var condEnd int + if i+1 < len(joinMatches) { + condEnd = joinMatches[i+1][0] + } else { + condEnd = len(joinBlock) + } + cond := strings.TrimSpace(joinBlock[condStart:condEnd]) + onConditions = append(onConditions, cond) + } + + // Split SET clause from WHERE clause + var setClause, whereClause string + whereIdx := regexp.MustCompile(`(?i)\sWHERE\s`).FindStringIndex(setAndWhere) + if whereIdx != nil { + setClause = strings.TrimSpace(setAndWhere[:whereIdx[0]]) + whereClause = strings.TrimSpace(setAndWhere[whereIdx[1]:]) + } else { + setClause = strings.TrimSpace(setAndWhere) + } + + allConditions := strings.Join(onConditions, " AND ") + if whereClause != "" { + allConditions += " AND " + whereClause + } + + // PG UPDATE SET requires bare column names — strip table/alias qualifiers + qualifier := alias1 + if qualifier == "" { + qualifier = table1 + } + setClause = regexp.MustCompile(`\b`+regexp.QuoteMeta(qualifier)+`\.(\w+)\s*=`). + ReplaceAllString(setClause, "$1 =") + + if alias1 != "" { + return fmt.Sprintf("UPDATE %s %s SET %s FROM %s WHERE %s", + table1, alias1, setClause, strings.Join(fromTables, ", "), allConditions) + } + return fmt.Sprintf("UPDATE %s SET %s FROM %s WHERE %s", + table1, setClause, strings.Join(fromTables, ", "), allConditions) +} + From 5e90159ff38726cfb61e6afface337b2dc7692d6 Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Tue, 31 Mar 2026 22:29:37 -0400 Subject: [PATCH 2/6] feat(datastore): wire PostgreSQL into Datastore + add test infrastructure - Add dialect field to Datastore struct and dialectForDriver() wiring - Add insertAndGetID / insertAndGetIDTx helpers (RETURNING id for PG) - Add dialectStep() to migration client for dual-dialect migrations - Add PG baseline schema (pg_baseline_schema.sql) for fresh PG databases - Add TruncateTables PG support and InsertAndGetLastID test helper - Wire dialect into AndroidDatastore - Add postgres_smoke_test.go (excluded from regular CI via //go:build ignore) --- server/datastore/mysql/android_mysql.go | 4 +- .../mysql/migrations/data/migration.go | 13 +- .../mysql/migrations/tables/migration.go | 72 +- server/datastore/mysql/mysql.go | 225 +- server/datastore/mysql/pg_baseline_schema.sql | 3461 +++++++++++++++++ server/datastore/mysql/postgres_smoke_test.go | 417 ++ server/datastore/mysql/schema.sql | 10 +- server/datastore/mysql/testing_utils.go | 260 +- 8 files changed, 4430 insertions(+), 32 deletions(-) create mode 100644 server/datastore/mysql/pg_baseline_schema.sql create mode 100644 server/datastore/mysql/postgres_smoke_test.go diff --git a/server/datastore/mysql/android_mysql.go b/server/datastore/mysql/android_mysql.go index 3f842777d7e..c393b2b5da7 100644 --- a/server/datastore/mysql/android_mysql.go +++ b/server/datastore/mysql/android_mysql.go @@ -17,14 +17,16 @@ type AndroidDatastore struct { logger *slog.Logger primary *sqlx.DB replica fleet.DBReader // so it cannot be used to perform writes + dialect DialectHelper } // NewAndroidDatastore creates a new Android Datastore -func NewAndroidDatastore(logger *slog.Logger, primary *sqlx.DB, replica fleet.DBReader) android.Datastore { +func NewAndroidDatastore(logger *slog.Logger, primary *sqlx.DB, replica fleet.DBReader, dialect DialectHelper) android.Datastore { return &AndroidDatastore{ logger: logger, primary: primary, replica: replica, + dialect: dialect, } } diff --git a/server/datastore/mysql/migrations/data/migration.go b/server/datastore/mysql/migrations/data/migration.go index 6185cc8328d..5df534d7880 100644 --- a/server/datastore/mysql/migrations/data/migration.go +++ b/server/datastore/mysql/migrations/data/migration.go @@ -1,5 +1,16 @@ package data -import "github.com/fleetdm/fleet/v4/server/goose" +import ( + "fmt" + + "github.com/fleetdm/fleet/v4/server/goose" +) var MigrationClient = goose.New("migration_status_data", goose.MySqlDialect{}) + +// SetDialect updates the migration client's SQL dialect. +func SetDialect(driver string) { + if err := MigrationClient.SetDialect(driver); err != nil { + panic(fmt.Sprintf("migrations/data: unsupported dialect %q: %v", driver, err)) + } +} diff --git a/server/datastore/mysql/migrations/tables/migration.go b/server/datastore/mysql/migrations/tables/migration.go index b8724086bdf..8a5791ec15d 100644 --- a/server/datastore/mysql/migrations/tables/migration.go +++ b/server/datastore/mysql/migrations/tables/migration.go @@ -18,6 +18,14 @@ import ( var MigrationClient = goose.New("migration_status_tables", goose.MySqlDialect{}) +// SetDialect updates the migration client's SQL dialect. +// Call before running migrations when using a non-MySQL database. +func SetDialect(driver string) { + if err := MigrationClient.SetDialect(driver); err != nil { + panic(fmt.Sprintf("migrations/tables: unsupported dialect %q: %v", driver, err)) + } +} + // can override in tests var ( outputTo io.Writer = os.Stderr @@ -105,7 +113,57 @@ func withSteps(steps []migrationStep, tx *sql.Tx) error { return nil } +// dialectStep returns a migrationStep that executes the MySQL statement +// for MySQL databases and the PostgreSQL statement for PostgreSQL databases. +// The driver is determined at migration time from the goose client's dialect. +// Pass an empty string for either statement to make it a no-op for that dialect. +func dialectStep(mysqlStmt, pgStmt string) migrationStep { + return func(tx *sql.Tx) error { + // Determine dialect from the connection's driver. + // In the current architecture, the migration client always uses MySqlDialect, + // so pgStmt will not be executed until a PostgreSQL migration client exists. + // For now, always execute mysqlStmt. + stmt := mysqlStmt + if stmt == "" { + return nil + } + _, err := tx.Exec(stmt) + return err + } +} + +// migrationHelper provides dialect-specific schema introspection for migrations. +// The default implementation uses MySQL information_schema. +// When PostgreSQL support is added, a pgMigrationHelper will use pg_catalog. +type migrationHelper interface { + fkExists(tx *sql.Tx, table, name string) bool + constraintExists(tx *sql.Tx, table, name string) bool + columnExists(tx *sql.Tx, table, column string) bool + columnsExists(tx *sql.Tx, table string, columns ...string) bool + tableExists(tx *sql.Tx, table string) bool +} + +// mysqlMigrationHelper implements migrationHelper using MySQL information_schema. +type mysqlMigrationHelper struct{} + +// defaultMigrationHelper is the migration helper used by all current migrations. +// It defaults to MySQL since that's the only supported database. +var defaultMigrationHelper migrationHelper = mysqlMigrationHelper{} + +// Package-level functions delegate to the default helper for backwards compatibility. func fkExists(tx *sql.Tx, table, name string) bool { + return defaultMigrationHelper.fkExists(tx, table, name) +} + +func constraintExists(tx *sql.Tx, table, name string) bool { + return defaultMigrationHelper.constraintExists(tx, table, name) +} + +func columnExists(tx *sql.Tx, table, column string) bool { + return defaultMigrationHelper.columnExists(tx, table, column) +} + +func (mysqlMigrationHelper) fkExists(tx *sql.Tx, table, name string) bool { var count int err := tx.QueryRow(` SELECT COUNT(1) @@ -121,7 +179,7 @@ AND CONSTRAINT_NAME = ? return count > 0 } -func constraintExists(tx *sql.Tx, table, name string) bool { +func (mysqlMigrationHelper) constraintExists(tx *sql.Tx, table, name string) bool { var count int err := tx.QueryRow(` SELECT COUNT(1) @@ -137,11 +195,15 @@ AND CONSTRAINT_NAME = ? return count > 0 } -func columnExists(tx *sql.Tx, table, column string) bool { - return columnsExists(tx, table, column) +func (mysqlMigrationHelper) columnExists(tx *sql.Tx, table, column string) bool { + return mysqlMigrationHelper{}.columnsExists(tx, table, column) } func columnsExists(tx *sql.Tx, table string, columns ...string) bool { + return defaultMigrationHelper.columnsExists(tx, table, columns...) +} + +func (mysqlMigrationHelper) columnsExists(tx *sql.Tx, table string, columns ...string) bool { if len(columns) == 0 { return false } @@ -173,6 +235,10 @@ WHERE } func tableExists(tx *sql.Tx, table string) bool { + return defaultMigrationHelper.tableExists(tx, table) +} + +func (mysqlMigrationHelper) tableExists(tx *sql.Tx, table string) bool { var count int err := tx.QueryRow( ` diff --git a/server/datastore/mysql/mysql.go b/server/datastore/mysql/mysql.go index c3a0a02ff81..230a7d7af8d 100644 --- a/server/datastore/mysql/mysql.go +++ b/server/datastore/mysql/mysql.go @@ -4,6 +4,7 @@ package mysql import ( "context" "database/sql" + _ "embed" "errors" "fmt" "log/slog" @@ -33,6 +34,7 @@ import ( scep_depot "github.com/fleetdm/fleet/v4/server/mdm/scep/depot" common_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql" "github.com/go-sql-driver/mysql" + _ "github.com/fleetdm/fleet/v4/server/platform/postgres" // register pgx-rebind driver for PostgreSQL "github.com/hashicorp/go-multierror" "github.com/jmoiron/sqlx" "go.opentelemetry.io/otel/attribute" @@ -59,10 +61,11 @@ type Datastore struct { replica fleet.DBReader // so it cannot be used to perform writes primary *sqlx.DB - logger *slog.Logger - clock clock.Clock - config config.MysqlConfig - pusher nano_push.Pusher + logger *slog.Logger + clock clock.Clock + config config.MysqlConfig + dialect DialectHelper + pusher nano_push.Pusher android.Datastore // nil if no read replica @@ -113,12 +116,58 @@ func (ds *Datastore) reader(ctx context.Context) fleet.DBReader { return ds.replica } +// currentDatabaseFn returns the SQL function to get the current database name. +// MySQL: DATABASE(), PostgreSQL: current_database() +func (ds *Datastore) currentDatabaseFn() string { + if ds.dialect.ReturningID() != "" { + return "current_database()" + } + return "(SELECT DATABASE())" +} + // writer returns the DB instance to use for write statements, which is always // the primary. func (ds *Datastore) writer(ctx context.Context) *sqlx.DB { return ds.primary } +// Querier is any type that can execute SQL (sqlx.DB, sqlx.Tx, sqlx.ExtContext). +type Querier interface { + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +// insertAndGetID executes an INSERT and returns the auto-generated ID. +// For MySQL, uses LastInsertId(). For PostgreSQL, appends RETURNING id. +func (ds *Datastore) insertAndGetID(ctx context.Context, q Querier, query string, args ...any) (int64, error) { + if ds.dialect.ReturningID() != "" { + // PostgreSQL: use RETURNING id + var id int64 + err := q.QueryRowContext(ctx, query+ds.dialect.ReturningID(), args...).Scan(&id) + return id, err + } + // MySQL: use LastInsertId + res, err := q.ExecContext(ctx, query, args...) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +// insertAndGetIDTx is like insertAndGetID but for sqlx.ExtContext (transactions). +func insertAndGetIDTx(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, query string, args ...any) (int64, error) { + if dialect.ReturningID() != "" { + var id int64 + err := tx.QueryRowxContext(ctx, query+dialect.ReturningID(), args...).Scan(&id) + return id, err + } + res, err := tx.ExecContext(ctx, query, args...) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + // loadOrPrepareStmt will load a statement from the statement cache. // If not available, it will attempt to prepare (create) it. // Returns nil if it failed to prepare a statement. @@ -241,6 +290,13 @@ func NewDBConnections(cfg config.MysqlConfig, opts ...DBOption) (*common_mysql.D if err := checkAndModifyConfig(&cfg); err != nil { return nil, err } + + // Set migration client dialects to match the configured driver. + if cfg.Driver == "postgres" { + tables.SetDialect("postgres") + data.SetDialect("postgres") + } + // Convert replica config once so that checkAndModifyConfig mutations are preserved for the later NewDB call. var replicaConf *config.MysqlConfig if options.ReplicaConfig != nil { @@ -286,12 +342,13 @@ func NewDatastore(conns *common_mysql.DBConnections, cfg config.MysqlConfig, c c logger: conns.Options.Logger, clock: c, config: cfg, + dialect: dialectForDriver(cfg.Driver), readReplicaConfig: conns.Options.ReplicaConfig, writeCh: make(chan itemToWrite), stmtCache: make(map[string]*sqlx.Stmt), minLastOpenedAtDiff: conns.Options.MinLastOpenedAtDiff, serverPrivateKey: conns.Options.PrivateKey, - Datastore: NewAndroidDatastore(conns.Options.Logger, conns.Primary, conns.Replica), + Datastore: NewAndroidDatastore(conns.Options.Logger, conns.Primary, conns.Replica, dialectForDriver(cfg.Driver)), } go ds.writeChanLoop() @@ -378,9 +435,43 @@ func init() { } func NewDB(conf *config.MysqlConfig, opts *common_mysql.DBOptions) (*sqlx.DB, error) { + if conf.Driver == "postgres" { + return newPostgresDB(conf) + } return common_mysql.NewDB(toCommonMysqlConfig(conf), opts, otelTracedDriverName) } +// newPostgresDB opens a PostgreSQL connection using pgx/stdlib. +func newPostgresDB(conf *config.MysqlConfig) (*sqlx.DB, error) { + // Build PostgreSQL DSN from the MySQL-style config fields. + // Address is expected as "host:port". + host, port, err := net.SplitHostPort(conf.Address) + if err != nil { + host = conf.Address + port = "5432" + } + dsn := fmt.Sprintf( + "host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", + host, port, conf.Username, conf.Password, conf.Database, + ) + if conf.TLSCA != "" { + dsn = fmt.Sprintf( + "host=%s port=%s user=%s password=%s dbname=%s sslmode=verify-ca sslrootcert=%s", + host, port, conf.Username, conf.Password, conf.Database, conf.TLSCA, + ) + } + + // Use "pgx-rebind" driver which wraps pgx/stdlib and auto-converts + // MySQL-style ? placeholders to PostgreSQL $N placeholders. + db, err := sqlx.Open("pgx-rebind", dsn) + if err != nil { + return nil, fmt.Errorf("open postgres: %w", err) + } + db.SetMaxOpenConns(conf.MaxOpenConns) + db.SetMaxIdleConns(conf.MaxIdleConns) + return db, nil +} + // toCommonMysqlConfig converts a config.MysqlConfig to common_mysql.MysqlConfig. func toCommonMysqlConfig(conf *config.MysqlConfig) *common_mysql.MysqlConfig { return &common_mysql.MysqlConfig{ @@ -439,7 +530,26 @@ func fromCommonMysqlConfig(conf *common_mysql.MysqlConfig) *config.MysqlConfig { } } +// dialectForDriver returns the DialectHelper for the given driver name. +// Empty string defaults to "mysql". +func dialectForDriver(driver string) DialectHelper { + switch driver { + case "postgres": + return postgresDialect{} + case "", "mysql": + return mysqlDialect{} + default: + // checkAndModifyConfig validates the driver before this is called, + // so reaching here means a programming error. + panic(fmt.Sprintf("unsupported database driver: %q", driver)) + } +} + func checkAndModifyConfig(conf *config.MysqlConfig) error { + if conf.Driver != "" && conf.Driver != "mysql" && conf.Driver != "postgres" { + return fmt.Errorf("unsupported database driver %q: valid values are \"mysql\" and \"postgres\"", conf.Driver) + } + if conf.PasswordPath != "" && conf.Password != "" { return errors.New("A MySQL password and a MySQL password file were provided - please specify only one") } @@ -488,13 +598,45 @@ func setupIAMAuthIfNeeded(conf *config.MysqlConfig, opts *common_mysql.DBOptions } func (ds *Datastore) MigrateTables(ctx context.Context) error { + if _, ok := ds.dialect.(postgresDialect); ok { + return ds.migratePGBaseline(ctx) + } return tables.MigrationClient.Up(ds.writer(ctx).DB, "") } func (ds *Datastore) MigrateData(ctx context.Context) error { + if _, ok := ds.dialect.(postgresDialect); ok { + // PG baseline schema includes all data migrations (label seeds, etc.) + return nil + } return data.MigrationClient.Up(ds.writer(ctx).DB, "") } +//go:embed pg_baseline_schema.sql +var pgBaselineSchemaSQL string + +// migratePGBaseline applies the PG baseline schema for fresh PostgreSQL databases. +// It checks if tables already exist and skips if so. +func (ds *Datastore) migratePGBaseline(ctx context.Context) error { + var exists bool + err := ds.writer(ctx).GetContext(ctx, &exists, + `SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'hosts')`) + if err != nil { + return fmt.Errorf("checking PG schema: %w", err) + } + if exists { + ds.logger.InfoContext(ctx, "PostgreSQL schema already exists, skipping baseline") + return nil + } + ds.logger.InfoContext(ctx, "Applying PostgreSQL baseline schema") + _, err = ds.writer(ctx).ExecContext(ctx, pgBaselineSchemaSQL) + if err != nil { + return fmt.Errorf("applying PG baseline schema: %w", err) + } + ds.logger.InfoContext(ctx, "PostgreSQL baseline schema applied successfully") + return nil +} + // loadMigrations manually loads the applied migrations in ascending // order (goose doesn't provide such functionality). // @@ -532,6 +674,20 @@ func (ds *Datastore) loadMigrations( // // It assumes some deployments may have performed migrations out of order. func (ds *Datastore) MigrationStatus(ctx context.Context) (*fleet.MigrationStatus, error) { + // For PostgreSQL, the baseline schema is applied atomically — either it's all there or not. + if _, ok := ds.dialect.(postgresDialect); ok { + var exists bool + err := ds.primary.GetContext(ctx, &exists, + `SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'hosts')`) + if err != nil { + return nil, fmt.Errorf("checking PG schema: %w", err) + } + if exists { + return &fleet.MigrationStatus{StatusCode: fleet.AllMigrationsCompleted}, nil + } + return &fleet.MigrationStatus{StatusCode: fleet.NoMigrationsCompleted}, nil + } + if tables.MigrationClient.Migrations == nil || data.MigrationClient.Migrations == nil { return nil, errors.New("unexpected nil migrations list") } @@ -732,14 +888,23 @@ func (ds *Datastore) HealthCheck() error { // Check that the primary is reachable and not in read-only mode. // After an AWS Aurora failover the old writer is demoted to a reader; // detecting this lets the health check fail so the orchestrator can restart Fleet. - var readOnly int - if err := ds.primary.QueryRowContext(context.Background(), "SELECT @@read_only").Scan(&readOnly); err != nil { - return err - } - if readOnly == 1 { - // Intentionally return an error so that the health check endpoint returns a 500, - // signaling the orchestrator (ECS, Kubernetes) to restart Fleet with fresh DB connections. - return errors.New("primary database is read-only, possible failover detected") + if _, ok := ds.dialect.(postgresDialect); ok { + // PG: check if the server is in recovery (read-only replica) + var inRecovery bool + if err := ds.primary.QueryRowContext(context.Background(), "SELECT pg_is_in_recovery()").Scan(&inRecovery); err != nil { + return err + } + if inRecovery { + return errors.New("primary database is in recovery (read-only), possible failover detected") + } + } else { + var readOnly int + if err := ds.primary.QueryRowContext(context.Background(), "SELECT @@read_only").Scan(&readOnly); err != nil { + return err + } + if readOnly == 1 { + return errors.New("primary database is read-only, possible failover detected") + } } if ds.readReplicaConfig != nil { @@ -1255,6 +1420,10 @@ func (ds *Datastore) ProcessList(ctx context.Context) ([]fleet.MySQLProcess, err return processList, nil } +// insertOnDuplicateDidInsertOrUpdate returns true if an INSERT ON DUPLICATE KEY +// UPDATE actually inserted or updated a row (vs no-op). +// MySQL: checks LastInsertId (non-zero on insert) AND RowsAffected (> 0). +// PostgreSQL: LastInsertId is not available, so just checks RowsAffected > 0. func insertOnDuplicateDidInsertOrUpdate(res sql.Result) bool { // From mysql's documentation: // @@ -1281,9 +1450,13 @@ func insertOnDuplicateDidInsertOrUpdate(res sql.Result) bool { // already holds: // https://github.com/go-sql-driver/mysql/blob/bcc459a906419e2890a50fc2c99ea6dd927a88f2/result.go - lastID, _ := res.LastInsertId() aff, _ := res.RowsAffected() - // something was updated (lastID != 0) AND row was found (aff == 1 or higher if more rows were found) + lastID, err := res.LastInsertId() + if err != nil { + // PostgreSQL doesn't support LastInsertId — fall back to RowsAffected only + return aff > 0 + } + // MySQL: something was inserted (lastID != 0) AND row was found (aff > 0) return lastID != 0 && aff > 0 } @@ -1321,9 +1494,9 @@ func (ds *Datastore) optimisticGetOrInsertWithWriter(ctx context.Context, writer if err != nil { if errors.Is(err, sql.ErrNoRows) { // this does not exist yet, try to insert it - res, err := writer.ExecContext(ctx, insertStmt.Statement, insertStmt.Args...) + insertedID, err := insertAndGetIDTx(ctx, writer, ds.dialect, insertStmt.Statement, insertStmt.Args...) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { // it might've been created between the select and the insert, read // again this time from the primary database connection. id, err := readID(writer) @@ -1334,8 +1507,7 @@ func (ds *Datastore) optimisticGetOrInsertWithWriter(ctx context.Context, writer } return 0, ctxerr.Wrap(ctx, err, "insert") } - id, _ := res.LastInsertId() - return uint(id), nil //nolint:gosec // dismiss G115 + return uint(insertedID), nil //nolint:gosec // dismiss G115 } return 0, ctxerr.Wrap(ctx, err, "get id from reader") } @@ -1395,3 +1567,18 @@ func batchProcessDB[T any]( } return nil } + +// jsonObjectFunc returns the SQL function name for building JSON objects. +// MySQL: JSON_OBJECT, PostgreSQL: jsonb_build_object +func (ds *Datastore) jsonObjectFunc() string { + if ds.dialect.ReturningID() != "" { + return "jsonb_build_object" + } + return "JSON_OBJECT" +} + +// resolveJSONFunc replaces %JSON_OBJECT% placeholder with the dialect-specific +// JSON object builder function name. +func (ds *Datastore) resolveJSONFunc(sql string) string { + return strings.ReplaceAll(sql, "%JSON_OBJECT%", ds.jsonObjectFunc()) +} diff --git a/server/datastore/mysql/pg_baseline_schema.sql b/server/datastore/mysql/pg_baseline_schema.sql new file mode 100644 index 00000000000..b498ac1f7cf --- /dev/null +++ b/server/datastore/mysql/pg_baseline_schema.sql @@ -0,0 +1,3461 @@ +-- Fleet PostgreSQL Test Baseline Schema +-- Auto-generated from MySQL test schema +-- Auto-generated from MySQL test schema + +CREATE TABLE IF NOT EXISTS "abm_tokens" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "organization_name" varchar(255) NOT NULL, + "apple_id" varchar(255) NOT NULL, + "terms_expired" boolean NOT NULL DEFAULT FALSE, + "renew_at" timestamp NOT NULL, + "token" bytea NOT NULL, + "macos_default_team_id" int DEFAULT NULL, + "ios_default_team_id" int DEFAULT NULL, + "ipados_default_team_id" int DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_abm_tokens_organization_name" UNIQUE ("organization_name") +); + +CREATE TABLE IF NOT EXISTS "activities" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "user_id" int DEFAULT NULL, + "user_name" varchar(255) DEFAULT NULL, + "activity_type" varchar(255) NOT NULL, + "details" jsonb DEFAULT NULL, + "streamed" boolean NOT NULL DEFAULT FALSE, + "user_email" varchar(255) NOT NULL DEFAULT '', + "fleet_initiated" boolean NOT NULL DEFAULT FALSE, + "host_only" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "activity_past" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "user_id" int DEFAULT NULL, + "user_name" varchar(255) DEFAULT NULL, + "activity_type" varchar(255) NOT NULL, + "details" jsonb DEFAULT NULL, + "streamed" boolean NOT NULL DEFAULT FALSE, + "user_email" varchar(255) NOT NULL DEFAULT '', + "fleet_initiated" boolean NOT NULL DEFAULT FALSE, + "host_only" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "activity_host_past" ( + "host_id" int NOT NULL, + "activity_id" int NOT NULL, + PRIMARY KEY ("host_id","activity_id") +); + +CREATE TABLE IF NOT EXISTS "aggregated_stats" ( + "id" bigint NOT NULL, + "type" varchar(255) NOT NULL, + "json_value" jsonb NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "global_stats" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id","type","global_stats") +); + +CREATE TABLE IF NOT EXISTS "android_app_configurations" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "application_id" varchar(255) NOT NULL, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "configuration" jsonb NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_global_or_team_id_application_id" UNIQUE ("global_or_team_id","application_id") +); + +CREATE TABLE IF NOT EXISTS "android_devices" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_id" int NOT NULL, + "device_id" varchar(32) NOT NULL, + "enterprise_specific_id" varchar(64) DEFAULT NULL, + "last_policy_sync_time" timestamp DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "applied_policy_id" varchar(100) DEFAULT NULL, + "applied_policy_version" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_android_devices_host_id" UNIQUE ("host_id"), + CONSTRAINT "idx_android_devices_device_id" UNIQUE ("device_id"), + CONSTRAINT "idx_android_devices_enterprise_specific_id" UNIQUE ("enterprise_specific_id") +); + +CREATE TABLE IF NOT EXISTS "android_enterprises" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "signup_name" varchar(63) NOT NULL DEFAULT '', + "enterprise_id" varchar(63) NOT NULL DEFAULT '', + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP , + "signup_token" varchar(64) NOT NULL DEFAULT '', + "pubsub_topic_id" varchar(64) NOT NULL DEFAULT '', + "user_id" int NOT NULL DEFAULT '0', + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "android_policy_requests" ( + "request_uuid" varchar(36) NOT NULL, + "request_name" varchar(255) NOT NULL, + "policy_id" varchar(100) NOT NULL, + "payload" jsonb NOT NULL, + "status_code" int NOT NULL, + "error_details" text, + "applied_policy_version" int DEFAULT NULL, + "policy_version" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("request_uuid") +); + +CREATE TABLE IF NOT EXISTS "app_config_json" ( + "id" int NOT NULL DEFAULT '1', + "json_value" jsonb NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "batch_activities" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "script_id" int NOT NULL, + "execution_id" varchar(255) NOT NULL, + "user_id" int DEFAULT NULL, + "job_id" int DEFAULT NULL, + "status" varchar(255) DEFAULT NULL, + "activity_type" varchar(255) DEFAULT NULL, + "num_targeted" int DEFAULT NULL, + "num_pending" int DEFAULT NULL, + "num_ran" int DEFAULT NULL, + "num_errored" int DEFAULT NULL, + "num_incompatible" int DEFAULT NULL, + "num_canceled" int DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "started_at" timestamp DEFAULT NULL, + "finished_at" timestamp DEFAULT NULL, + "canceled" boolean DEFAULT FALSE, + PRIMARY KEY ("id"), + CONSTRAINT "idx_batch_script_executions_execution_id" UNIQUE ("execution_id") +); + +CREATE TABLE IF NOT EXISTS "batch_activity_host_results" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "batch_execution_id" varchar(255) NOT NULL, + "host_id" int NOT NULL, + "host_execution_id" varchar(255) DEFAULT NULL, + "error" varchar(255) DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "unique_batch_host_results_execution_hostid" UNIQUE ("batch_execution_id","host_id") +); + +CREATE TABLE IF NOT EXISTS "ca_config_assets" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "type" text NOT NULL, + "name" varchar(255) NOT NULL, + "value" bytea NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_ca_config_assets_name" UNIQUE ("name") +); + +CREATE TABLE IF NOT EXISTS "calendar_events" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "email" varchar(255) NOT NULL, + "start_time" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "end_time" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "event" jsonb NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "timezone" varchar(64) DEFAULT NULL, + "uuid_bin" bytea NOT NULL, + "uuid" text NOT NULL DEFAULT '', + PRIMARY KEY ("id"), + CONSTRAINT "idx_one_calendar_event_per_email" UNIQUE ("email"), + CONSTRAINT "idx_calendar_events_uuid_bin_unique" UNIQUE ("uuid_bin") +); + +CREATE TABLE IF NOT EXISTS "carve_blocks" ( + "metadata_id" int NOT NULL, + "block_id" int NOT NULL, + "data" bytea, + PRIMARY KEY ("metadata_id","block_id") +); + +CREATE TABLE IF NOT EXISTS "carve_metadata" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_id" int NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "name" varchar(255) DEFAULT NULL, + "block_count" int NOT NULL, + "block_size" int NOT NULL, + "carve_size" bigint NOT NULL, + "carve_id" varchar(64) NOT NULL, + "request_id" varchar(64) NOT NULL, + "session_id" varchar(255) NOT NULL, + "expired" boolean DEFAULT FALSE, + "max_block" int DEFAULT '-1', + "error" text, + PRIMARY KEY ("id"), + CONSTRAINT "idx_session_id" UNIQUE ("session_id"), + CONSTRAINT "idx_name" UNIQUE ("name") +); + +CREATE TABLE IF NOT EXISTS "certificate_authorities" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "type" text NOT NULL, + "name" varchar(255) NOT NULL, + "url" text NOT NULL, + "api_token_encrypted" bytea, + "profile_id" varchar(255) DEFAULT NULL, + "certificate_common_name" varchar(255) DEFAULT NULL, + "certificate_user_principal_names" jsonb DEFAULT NULL, + "certificate_seat_id" varchar(255) DEFAULT NULL, + "admin_url" text, + "username" varchar(255) DEFAULT NULL, + "password_encrypted" bytea, + "challenge_url" text, + "challenge_encrypted" bytea, + "client_id" varchar(255) DEFAULT NULL, + "client_secret_encrypted" bytea, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_ca_type_name" UNIQUE ("type","name") +); + +CREATE TABLE IF NOT EXISTS "certificate_templates" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "team_id" int NOT NULL, + "certificate_authority_id" int NOT NULL, + "name" varchar(255) NOT NULL, + "subject_name" text NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_cert_team_name" UNIQUE ("team_id","name") +); + +CREATE TABLE IF NOT EXISTS "challenges" ( + "challenge" char(32) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("challenge") +); + +CREATE TABLE IF NOT EXISTS "conditional_access_scep_certificates" ( + "serial" bigint NOT NULL, + "host_id" int NOT NULL, + "name" varchar(64) NOT NULL, + "not_valid_before" timestamptz NOT NULL, + "not_valid_after" timestamptz NOT NULL, + "certificate_pem" text NOT NULL, + "revoked" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamptz DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamptz DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "conditional_access_scep_certificates_chk_1" CHECK ((substr("certificate_pem",1,27) = '-----BEGIN CERTIFICATE-----')), + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "conditional_access_scep_serials" ( + "serial" bigint GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "cron_stats" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "name" varchar(255) NOT NULL, + "instance" varchar(255) NOT NULL, + "stats_type" varchar(255) NOT NULL, + "status" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "errors" jsonb DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "cve_meta" ( + "cve" varchar(20) NOT NULL, + "cvss_score" double precision DEFAULT NULL, + "epss_probability" double precision DEFAULT NULL, + "cisa_known_exploit" boolean DEFAULT NULL, + "published" timestamp NULL DEFAULT NULL, + "description" text, + PRIMARY KEY ("cve") +); + +CREATE TABLE IF NOT EXISTS "default_team_config_json" ( + "id" int NOT NULL DEFAULT '1', + "json_value" jsonb NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "default_team_config_id" CHECK (("id" = 1)), + PRIMARY KEY ("id"), + CONSTRAINT "id" UNIQUE ("id") +); + +CREATE TABLE IF NOT EXISTS "distributed_query_campaign_targets" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "type" int DEFAULT NULL, + "distributed_query_campaign_id" int DEFAULT NULL, + "target_id" int DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "distributed_query_campaigns" ( + "id" bigint NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "query_id" int DEFAULT NULL, + "status" int DEFAULT NULL, + "user_id" int DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "email_changes" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "user_id" int NOT NULL, + "token" varchar(128) NOT NULL, + "new_email" varchar(255) NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_email_changes_token" UNIQUE ("token") +); + +CREATE TABLE IF NOT EXISTS "enroll_secrets" ( + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "secret" varchar(255) NOT NULL, + "team_id" int DEFAULT NULL, + PRIMARY KEY ("secret") +); + +CREATE TABLE IF NOT EXISTS "eulas" ( + "id" int NOT NULL, + "token" varchar(36) DEFAULT NULL, + "name" varchar(255) DEFAULT NULL, + "bytes" bytea, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "sha256" bytea DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "fleet_maintained_apps" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "name" varchar(255) NOT NULL, + "slug" varchar(255) NOT NULL, + "platform" varchar(255) NOT NULL, + "unique_identifier" varchar(255) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_fleet_library_apps_token" UNIQUE ("slug") +); + +CREATE TABLE IF NOT EXISTS "fleet_variables" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "name" varchar(255) NOT NULL DEFAULT '', + "is_prefix" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_fleet_variables_name_is_prefix" UNIQUE ("name","is_prefix") +); + +CREATE TABLE IF NOT EXISTS "host_activities" ( + "host_id" int NOT NULL, + "activity_id" int NOT NULL, + PRIMARY KEY ("host_id","activity_id") +); + +CREATE TABLE IF NOT EXISTS "host_additional" ( + "host_id" int NOT NULL, + "additional" jsonb DEFAULT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_batteries" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_id" int NOT NULL, + "serial_number" varchar(255) NOT NULL, + "cycle_count" int NOT NULL, + "health" varchar(40) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_batteries_host_id_serial_number" UNIQUE ("host_id","serial_number") +); + +CREATE TABLE IF NOT EXISTS "host_calendar_events" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_id" int NOT NULL, + "calendar_event_id" int NOT NULL, + "webhook_status" smallint NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_one_calendar_event_per_host" UNIQUE ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_certificate_sources" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "host_certificate_id" bigint NOT NULL, + "source" text NOT NULL, + "username" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_certificate_sources_unique" UNIQUE ("host_certificate_id","source","username") +); + +CREATE TABLE IF NOT EXISTS "host_certificate_templates" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_uuid" varchar(255) NOT NULL, + "certificate_template_id" int NOT NULL, + "fleet_challenge" varchar(32) DEFAULT NULL, + "status" varchar(20) NOT NULL DEFAULT 'pending', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "detail" text, + "operation_type" varchar(20) NOT NULL DEFAULT 'install', + "name" varchar(255) NOT NULL, + "uuid" uuid DEFAULT NULL, + "not_valid_before" timestamp DEFAULT NULL, + "not_valid_after" timestamp DEFAULT NULL, + "serial" varchar(40) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_certificate_templates_host_template" UNIQUE ("host_uuid","certificate_template_id") +); + +CREATE TABLE IF NOT EXISTS "host_certificates" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "host_id" int NOT NULL, + "not_valid_after" timestamp NOT NULL, + "not_valid_before" timestamp NOT NULL, + "certificate_authority" boolean NOT NULL, + "common_name" varchar(255) NOT NULL, + "key_algorithm" varchar(255) NOT NULL, + "key_strength" int NOT NULL, + "key_usage" varchar(255) NOT NULL, + "serial" varchar(255) NOT NULL, + "signing_algorithm" varchar(255) NOT NULL, + "subject_country" varchar(32) NOT NULL, + "subject_org" varchar(255) NOT NULL, + "subject_org_unit" varchar(255) NOT NULL, + "subject_common_name" varchar(255) NOT NULL, + "issuer_country" varchar(32) NOT NULL, + "issuer_org" varchar(255) NOT NULL, + "issuer_org_unit" varchar(255) NOT NULL, + "issuer_common_name" varchar(255) NOT NULL, + "sha1_sum" bytea NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "deleted_at" timestamp DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "host_conditional_access" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_id" int NOT NULL, + "bypassed_at" timestamp NULL DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_conditional_access_host_id" UNIQUE ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_dep_assignments" ( + "host_id" int NOT NULL, + "added_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "deleted_at" timestamp NULL DEFAULT NULL, + "profile_uuid" varchar(37) DEFAULT NULL, + "assign_profile_response" varchar(15) DEFAULT NULL, + "response_updated_at" timestamp NULL DEFAULT NULL, + "retry_job_id" int NOT NULL DEFAULT '0', + "abm_token_id" int DEFAULT NULL, + "mdm_migration_deadline" timestamp NULL DEFAULT NULL, + "mdm_migration_completed" timestamp NULL DEFAULT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_device_auth" ( + "host_id" int NOT NULL, + "token" varchar(255) NOT NULL, + "previous_token" varchar(255) DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("host_id"), + CONSTRAINT "idx_host_device_auth_token" UNIQUE ("token") +); + +CREATE TABLE IF NOT EXISTS "host_disk_encryption_keys" ( + "host_id" int NOT NULL, + "base64_encrypted" text NOT NULL, + "base64_encrypted_salt" varchar(255) NOT NULL DEFAULT '', + "key_slot" smallint DEFAULT NULL, + "decryptable" boolean DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "reset_requested" boolean NOT NULL DEFAULT FALSE, + "client_error" varchar(255) NOT NULL DEFAULT '', + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_disk_encryption_keys_archive" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "host_id" int NOT NULL, + "hardware_serial" varchar(255) NOT NULL DEFAULT '', + "base64_encrypted" text NOT NULL, + "base64_encrypted_salt" varchar(255) NOT NULL DEFAULT '', + "key_slot" smallint DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "host_disks" ( + "host_id" int NOT NULL, + "gigs_disk_space_available" decimal(10,2) NOT NULL DEFAULT '0.00', + "percent_disk_space_available" decimal(10,2) NOT NULL DEFAULT '0.00', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "encrypted" boolean DEFAULT NULL, + "gigs_total_disk_space" decimal(10,2) NOT NULL DEFAULT '0.00', + "tpm_pin_set" boolean DEFAULT FALSE, + "gigs_all_disk_space" decimal(10,2) DEFAULT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_display_names" ( + "host_id" int NOT NULL, + "display_name" varchar(255) NOT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_emails" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_id" int NOT NULL, + "email" varchar(255) NOT NULL, + "source" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "host_identity_scep_certificates" ( + "serial" bigint NOT NULL, + "host_id" int DEFAULT NULL, + "name" varchar(255) NOT NULL, + "not_valid_before" timestamptz NOT NULL, + "not_valid_after" timestamptz NOT NULL, + "certificate_pem" text NOT NULL, + "public_key_raw" bytea NOT NULL, + "revoked" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamptz DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamptz DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "host_identity_scep_certificates_chk_1" CHECK ((substr("certificate_pem",1,27) = '-----BEGIN CERTIFICATE-----')), + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "host_identity_scep_serials" ( + "serial" bigint GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "host_in_house_software_installs" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_id" int NOT NULL, + "in_house_app_id" int NOT NULL, + "command_uuid" varchar(127) NOT NULL, + "user_id" int DEFAULT NULL, + "platform" varchar(10) NOT NULL, + "removed" boolean NOT NULL DEFAULT FALSE, + "canceled" boolean NOT NULL DEFAULT FALSE, + "verification_command_uuid" varchar(127) DEFAULT NULL, + "verification_at" timestamp DEFAULT NULL, + "verification_failed_at" timestamp DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "self_service" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_in_house_software_installs_command_uuid" UNIQUE ("command_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_issues" ( + "host_id" int NOT NULL, + "failing_policies_count" int NOT NULL DEFAULT '0', + "critical_vulnerabilities_count" int NOT NULL DEFAULT '0', + "total_issues_count" int NOT NULL DEFAULT '0', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_last_known_locations" ( + "host_id" int NOT NULL, + "latitude" decimal(10,8) DEFAULT NULL, + "longitude" decimal(11,8) DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_mdm" ( + "host_id" int NOT NULL, + "enrolled" boolean NOT NULL DEFAULT FALSE, + "server_url" varchar(255) NOT NULL DEFAULT '', + "installed_from_dep" boolean NOT NULL DEFAULT FALSE, + "mdm_id" int DEFAULT NULL, + "is_server" boolean DEFAULT NULL, + "fleet_enroll_ref" varchar(36) NOT NULL DEFAULT '', + "enrollment_status" text, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "is_personal_enrollment" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_actions" ( + "host_id" int NOT NULL, + "lock_ref" varchar(36) DEFAULT NULL, + "wipe_ref" varchar(36) DEFAULT NULL, + "unlock_pin" varchar(6) DEFAULT NULL, + "unlock_ref" varchar(36) DEFAULT NULL, + "fleet_platform" varchar(255) NOT NULL DEFAULT '', + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_android_profiles" ( + "host_uuid" varchar(255) NOT NULL, + "status" varchar(20) DEFAULT NULL, + "operation_type" varchar(20) DEFAULT NULL, + "detail" text, + "profile_uuid" varchar(37) NOT NULL DEFAULT '', + "profile_name" varchar(255) NOT NULL DEFAULT '', + "policy_request_uuid" varchar(36) DEFAULT NULL, + "device_request_uuid" varchar(36) DEFAULT NULL, + "request_fail_count" smallint NOT NULL DEFAULT '0', + "included_in_policy_version" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "can_reverify" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("host_uuid","profile_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_apple_awaiting_configuration" ( + "host_uuid" varchar(255) NOT NULL, + "awaiting_configuration" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("host_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_apple_bootstrap_packages" ( + "host_uuid" varchar(127) NOT NULL, + "command_uuid" varchar(127) DEFAULT NULL, + "skipped" boolean NOT NULL DEFAULT FALSE, + CONSTRAINT "ck_skipped_or_commanduuid" CHECK ((("skipped" = false) = ("command_uuid" is not null))), + PRIMARY KEY ("host_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_apple_declarations" ( + "host_uuid" varchar(255) NOT NULL, + "status" varchar(20) DEFAULT NULL, + "operation_type" varchar(20) DEFAULT NULL, + "detail" text, + "token" bytea NOT NULL, + "declaration_uuid" varchar(37) NOT NULL DEFAULT '', + "declaration_identifier" varchar(255) NOT NULL, + "declaration_name" varchar(255) NOT NULL DEFAULT '', + "secrets_updated_at" timestamp DEFAULT NULL, + "resync" boolean NOT NULL DEFAULT FALSE, + "scope" text NOT NULL DEFAULT 'System', + PRIMARY KEY ("host_uuid","declaration_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_apple_profiles" ( + "profile_identifier" varchar(255) NOT NULL, + "host_uuid" varchar(255) NOT NULL, + "status" varchar(20) DEFAULT NULL, + "operation_type" varchar(20) DEFAULT NULL, + "detail" text, + "command_uuid" varchar(127) NOT NULL, + "profile_name" varchar(255) NOT NULL DEFAULT '', + "checksum" bytea NOT NULL, + "retries" smallint NOT NULL DEFAULT '0', + "profile_uuid" varchar(37) NOT NULL DEFAULT '', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "secrets_updated_at" timestamp DEFAULT NULL, + "ignore_error" boolean NOT NULL DEFAULT FALSE, + "variables_updated_at" timestamp DEFAULT NULL, + "scope" text NOT NULL DEFAULT 'System', + PRIMARY KEY ("host_uuid","profile_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_commands" ( + "host_id" int NOT NULL, + "command_type" varchar(31) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("host_id","command_type") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_idp_accounts" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_uuid" varchar(255) NOT NULL, + "account_uuid" varchar(36) NOT NULL DEFAULT '', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_mdm_idp_accounts" UNIQUE ("host_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_managed_certificates" ( + "host_uuid" varchar(255) NOT NULL, + "profile_uuid" varchar(37) NOT NULL, + "type" text NOT NULL DEFAULT 'ndes', + "ca_name" varchar(255) NOT NULL DEFAULT 'NDES', + "challenge_retrieved_at" timestamp NULL DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "not_valid_after" timestamp DEFAULT NULL, + "serial" varchar(40) DEFAULT NULL, + "not_valid_before" timestamp DEFAULT NULL, + PRIMARY KEY ("host_uuid","profile_uuid","ca_name") +); + +CREATE TABLE IF NOT EXISTS "host_mdm_windows_profiles" ( + "host_uuid" varchar(255) NOT NULL, + "status" varchar(20) DEFAULT NULL, + "operation_type" varchar(20) DEFAULT NULL, + "detail" text, + "command_uuid" varchar(127) NOT NULL, + "profile_name" varchar(255) NOT NULL DEFAULT '', + "retries" smallint NOT NULL DEFAULT '0', + "profile_uuid" varchar(37) NOT NULL DEFAULT '', + "checksum" bytea NOT NULL DEFAULT ''::bytea, + "secrets_updated_at" timestamp DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("host_uuid","profile_uuid") +); + +CREATE TABLE IF NOT EXISTS "host_munki_info" ( + "host_id" int NOT NULL, + "version" varchar(255) NOT NULL DEFAULT '', + "deleted_at" timestamp NULL DEFAULT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_munki_issues" ( + "host_id" int NOT NULL, + "munki_issue_id" int NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("host_id","munki_issue_id") +); + +CREATE TABLE IF NOT EXISTS "host_operating_system" ( + "host_id" int NOT NULL, + "os_id" int NOT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_orbit_info" ( + "host_id" int NOT NULL, + "version" varchar(50) NOT NULL, + "desktop_version" varchar(50) DEFAULT NULL, + "scripts_enabled" boolean DEFAULT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_scim_user" ( + "host_id" int NOT NULL, + "scim_user_id" int NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_script_results" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_id" int NOT NULL, + "execution_id" varchar(255) NOT NULL, + "output" text NOT NULL, + "runtime" int NOT NULL DEFAULT '0', + "exit_code" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "script_id" int DEFAULT NULL, + "user_id" int DEFAULT NULL, + "sync_request" boolean NOT NULL DEFAULT FALSE, + "script_content_id" int DEFAULT NULL, + "host_deleted_at" timestamp NULL DEFAULT NULL, + "timeout" int DEFAULT NULL, + "policy_id" int DEFAULT NULL, + "setup_experience_script_id" int DEFAULT NULL, + "is_internal" boolean DEFAULT FALSE, + "canceled" boolean NOT NULL DEFAULT FALSE, + "attempt_number" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_script_results_execution_id" UNIQUE ("execution_id") +); + +CREATE TABLE IF NOT EXISTS "host_seen_times" ( + "host_id" int NOT NULL, + "seen_time" timestamp NULL DEFAULT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_software" ( + "host_id" int NOT NULL, + "software_id" bigint NOT NULL, + "last_opened_at" timestamp NULL DEFAULT NULL, + PRIMARY KEY ("host_id","software_id") +); + +CREATE TABLE IF NOT EXISTS "host_software_installed_paths" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "host_id" int NOT NULL, + "software_id" bigint NOT NULL, + "installed_path" text NOT NULL, + "team_identifier" varchar(10) NOT NULL DEFAULT '', + "cdhash_sha256" char(64) DEFAULT NULL, + "executable_sha256" char(64) DEFAULT NULL, + "executable_path" text, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "host_software_installs" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "execution_id" varchar(255) NOT NULL, + "host_id" int NOT NULL, + "software_installer_id" int DEFAULT NULL, + "pre_install_query_output" text, + "install_script_output" text, + "install_script_exit_code" int DEFAULT NULL, + "post_install_script_output" text, + "post_install_script_exit_code" int DEFAULT NULL, + "user_id" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "self_service" boolean NOT NULL DEFAULT FALSE, + "host_deleted_at" timestamp NULL DEFAULT NULL, + "removed" boolean NOT NULL DEFAULT FALSE, + "uninstall_script_output" text, + "uninstall_script_exit_code" int DEFAULT NULL, + "uninstall" boolean NOT NULL DEFAULT FALSE, + "status" text, + "policy_id" int DEFAULT NULL, + "installer_filename" varchar(255) NOT NULL DEFAULT '[deleted installer]', + "version" varchar(255) NOT NULL DEFAULT 'unknown', + "software_title_id" int DEFAULT NULL, + "software_title_name" varchar(255) NOT NULL DEFAULT '[deleted title]', + "execution_status" text, + "canceled" boolean NOT NULL DEFAULT FALSE, + "attempt_number" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_software_installs_execution_id" UNIQUE ("execution_id") +); + +CREATE TABLE IF NOT EXISTS "host_updates" ( + "host_id" int NOT NULL, + "software_updated_at" timestamp NULL DEFAULT NULL, + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "host_users" ( + "host_id" int NOT NULL, + "uid" int NOT NULL, + "username" varchar(255) NOT NULL, + "groupname" varchar(255) DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "removed_at" timestamp NULL DEFAULT NULL, + "user_type" varchar(255) DEFAULT NULL, + "shell" varchar(255) DEFAULT '', + PRIMARY KEY ("host_id","uid","username") +); + +CREATE TABLE IF NOT EXISTS "host_vpp_software_installs" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_id" int NOT NULL, + "adam_id" varchar(255) NOT NULL, + "command_uuid" varchar(127) NOT NULL, + "user_id" int DEFAULT NULL, + "self_service" boolean NOT NULL DEFAULT FALSE, + "associated_event_id" varchar(36) DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "platform" varchar(10) NOT NULL, + "removed" boolean NOT NULL DEFAULT FALSE, + "vpp_token_id" int DEFAULT NULL, + "policy_id" int DEFAULT NULL, + "canceled" boolean NOT NULL DEFAULT FALSE, + "verification_command_uuid" varchar(127) DEFAULT NULL, + "verification_at" timestamp DEFAULT NULL, + "verification_failed_at" timestamp DEFAULT NULL, + "retry_count" int NOT NULL DEFAULT '0', + PRIMARY KEY ("id"), + CONSTRAINT "idx_host_vpp_software_installs_command_uuid" UNIQUE ("command_uuid") +); + +CREATE TABLE IF NOT EXISTS "hosts" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "osquery_host_id" varchar(255) DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "detail_updated_at" timestamp NULL DEFAULT NULL, + "node_key" varchar(255) DEFAULT NULL, + "hostname" varchar(255) NOT NULL DEFAULT '', + "uuid" varchar(255) NOT NULL DEFAULT '', + "platform" varchar(255) NOT NULL DEFAULT '', + "osquery_version" varchar(255) NOT NULL DEFAULT '', + "os_version" varchar(255) NOT NULL DEFAULT '', + "build" varchar(255) NOT NULL DEFAULT '', + "platform_like" varchar(255) NOT NULL DEFAULT '', + "code_name" varchar(255) NOT NULL DEFAULT '', + "uptime" bigint NOT NULL DEFAULT '0', + "memory" bigint NOT NULL DEFAULT '0', + "cpu_type" varchar(255) NOT NULL DEFAULT '', + "cpu_subtype" varchar(255) NOT NULL DEFAULT '', + "cpu_brand" varchar(255) NOT NULL DEFAULT '', + "cpu_physical_cores" int NOT NULL DEFAULT '0', + "cpu_logical_cores" int NOT NULL DEFAULT '0', + "hardware_vendor" varchar(255) NOT NULL DEFAULT '', + "hardware_model" varchar(255) NOT NULL DEFAULT '', + "hardware_version" varchar(255) NOT NULL DEFAULT '', + "hardware_serial" varchar(255) NOT NULL DEFAULT '', + "computer_name" varchar(255) NOT NULL DEFAULT '', + "primary_ip_id" int DEFAULT NULL, + "distributed_interval" int DEFAULT '0', + "logger_tls_period" int DEFAULT '0', + "config_tls_refresh" int DEFAULT '0', + "primary_ip" varchar(45) NOT NULL DEFAULT '', + "primary_mac" varchar(17) NOT NULL DEFAULT '', + "label_updated_at" timestamp NOT NULL DEFAULT '2000-01-01 00:00:00', + "last_enrolled_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "refetch_requested" boolean NOT NULL DEFAULT FALSE, + "team_id" int DEFAULT NULL, + "policy_updated_at" timestamp NOT NULL DEFAULT '2000-01-01 00:00:00', + "public_ip" varchar(45) NOT NULL DEFAULT '', + "orbit_node_key" varchar(255) DEFAULT NULL, + "refetch_critical_queries_until" timestamp NULL DEFAULT NULL, + "last_restarted_at" timestamp DEFAULT '0001-01-01 00:00:00.000000', + "timezone" varchar(255) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_osquery_host_id" UNIQUE ("osquery_host_id"), + CONSTRAINT "idx_host_unique_nodekey" UNIQUE ("node_key"), + CONSTRAINT "idx_host_unique_orbitnodekey" UNIQUE ("orbit_node_key") +); + +CREATE TABLE IF NOT EXISTS "in_house_app_labels" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "in_house_app_id" int NOT NULL, + "label_id" int NOT NULL, + "exclude" boolean NOT NULL DEFAULT FALSE, + "require_all" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "id_in_house_app_labels_in_house_app_id_label_id" UNIQUE ("in_house_app_id","label_id") +); + +CREATE TABLE IF NOT EXISTS "in_house_app_software_categories" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "software_category_id" int NOT NULL, + "in_house_app_id" int NOT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_in_house_app_id_software_category_id" UNIQUE ("in_house_app_id","software_category_id") +); + +CREATE TABLE IF NOT EXISTS "in_house_app_upcoming_activities" ( + "upcoming_activity_id" bigint NOT NULL, + "in_house_app_id" int NOT NULL, + "software_title_id" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("upcoming_activity_id") +); + +CREATE TABLE IF NOT EXISTS "in_house_apps" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "title_id" int DEFAULT NULL, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "filename" varchar(255) NOT NULL DEFAULT '', + "version" varchar(255) NOT NULL DEFAULT '', + "storage_id" varchar(64) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "platform" varchar(10) NOT NULL, + "bundle_identifier" varchar(255) NOT NULL DEFAULT '', + "self_service" boolean NOT NULL DEFAULT FALSE, + "url" varchar(4095) NOT NULL DEFAULT '', + PRIMARY KEY ("id"), + CONSTRAINT "global_or_team_id" UNIQUE ("global_or_team_id","filename","platform") +); + +CREATE TABLE IF NOT EXISTS "invite_teams" ( + "invite_id" int NOT NULL, + "team_id" int NOT NULL, + "role" varchar(64) NOT NULL, + PRIMARY KEY ("invite_id","team_id") +); + +CREATE TABLE IF NOT EXISTS "invites" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "invited_by" int NOT NULL, + "email" varchar(255) NOT NULL, + "name" varchar(255) DEFAULT NULL, + "position" varchar(255) DEFAULT NULL, + "token" varchar(255) NOT NULL, + "sso_enabled" boolean NOT NULL DEFAULT FALSE, + "global_role" varchar(64) DEFAULT NULL, + "mfa_enabled" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id"), + CONSTRAINT "idx_invite_unique_email" UNIQUE ("email"), + CONSTRAINT "idx_invite_unique_key" UNIQUE ("token") +); + +CREATE TABLE IF NOT EXISTS "jobs" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "name" varchar(255) NOT NULL, + "args" jsonb DEFAULT NULL, + "state" varchar(255) NOT NULL, + "retries" int NOT NULL DEFAULT '0', + "error" text, + "not_before" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "kernel_host_counts" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "software_title_id" int DEFAULT NULL, + "software_id" int DEFAULT NULL, + "os_version_id" int DEFAULT NULL, + "hosts_count" int NOT NULL, + "team_id" int NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_kernels_unique_mapping" UNIQUE ("os_version_id","team_id","software_id") +); + +CREATE TABLE IF NOT EXISTS "label_membership" ( + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "label_id" int NOT NULL, + "host_id" int NOT NULL, + PRIMARY KEY ("host_id","label_id") +); + +CREATE TABLE IF NOT EXISTS "labels" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "name" varchar(255) NOT NULL, + "description" varchar(255) NOT NULL DEFAULT '', + "query" text NOT NULL, + "platform" varchar(255) NOT NULL DEFAULT '', + "label_type" int NOT NULL DEFAULT '1', + "label_membership_type" int NOT NULL DEFAULT '0', + "author_id" int DEFAULT NULL, + "criteria" jsonb DEFAULT NULL, + "team_id" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_label_unique_name" UNIQUE ("name") +); + +CREATE TABLE IF NOT EXISTS "legacy_host_filevault_profiles" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_uuid" varchar(36) NOT NULL, + "status" varchar(20) NOT NULL, + "operation_type" varchar(20) NOT NULL, + "profile_uuid" varchar(37) NOT NULL, + "detail" text, + "command_uuid" varchar(127) NOT NULL, + "scope" text NOT NULL DEFAULT 'System', + "created_at" timestamp NOT NULL, + "updated_at" timestamp NOT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "legacy_host_mdm_enroll_refs" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_uuid" varchar(255) NOT NULL, + "enroll_ref" varchar(36) NOT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "legacy_host_mdm_idp_accounts" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_uuid" varchar(255) NOT NULL, + "email" varchar(255) NOT NULL, + "account_uuid" varchar(36) DEFAULT NULL, + "host_id" int DEFAULT NULL, + "email_id" int DEFAULT NULL, + "email_created_at" timestamp DEFAULT NULL, + "email_updated_at" timestamp DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "locks" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "name" varchar(255) DEFAULT NULL, + "owner" varchar(255) DEFAULT NULL, + "expires_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "locks_idx_name" UNIQUE ("name") +); + +CREATE TABLE IF NOT EXISTS "mdm_android_configuration_profiles" ( + "profile_uuid" varchar(37) NOT NULL DEFAULT '', + "team_id" int NOT NULL DEFAULT '0', + "name" varchar(255) NOT NULL, + "raw_json" jsonb NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("profile_uuid"), + CONSTRAINT "idx_mdm_android_configuration_profiles_team_id_name" UNIQUE ("team_id","name") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_bootstrap_packages" ( + "team_id" int NOT NULL, + "name" varchar(255) DEFAULT NULL, + "sha256" bytea NOT NULL, + "bytes" bytea, + "token" varchar(36) DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("team_id"), + CONSTRAINT "idx_token" UNIQUE ("token") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_configuration_profiles" ( + "profile_id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "team_id" int NOT NULL DEFAULT '0', + "identifier" varchar(255) NOT NULL, + "name" varchar(255) NOT NULL, + "mobileconfig" bytea NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp NULL DEFAULT NULL, + "checksum" bytea NOT NULL, + "profile_uuid" varchar(37) NOT NULL DEFAULT '', + "secrets_updated_at" timestamp DEFAULT NULL, + "scope" text NOT NULL DEFAULT 'System', + PRIMARY KEY ("profile_uuid"), + CONSTRAINT "idx_mdm_apple_config_prof_team_identifier" UNIQUE ("team_id","identifier"), + CONSTRAINT "idx_mdm_apple_config_prof_team_name" UNIQUE ("team_id","name"), + CONSTRAINT "idx_mdm_apple_config_prof_id" UNIQUE ("profile_id") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_declaration_activation_references" ( + "declaration_uuid" varchar(37) NOT NULL DEFAULT '', + "reference" varchar(37) NOT NULL DEFAULT '', + PRIMARY KEY ("declaration_uuid","reference") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_declarations" ( + "declaration_uuid" varchar(37) NOT NULL DEFAULT '', + "team_id" int NOT NULL DEFAULT '0', + "identifier" varchar(255) NOT NULL, + "name" varchar(255) NOT NULL, + "raw_json" text NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp NULL DEFAULT NULL, + "secrets_updated_at" timestamp DEFAULT NULL, + "token" bytea, + "scope" text NOT NULL DEFAULT 'System', + PRIMARY KEY ("declaration_uuid"), + CONSTRAINT "idx_mdm_apple_declaration_team_identifier" UNIQUE ("team_id","identifier"), + CONSTRAINT "idx_mdm_apple_declaration_team_name" UNIQUE ("team_id","name") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_declarative_requests" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "enrollment_id" varchar(255) NOT NULL, + "message_type" varchar(255) NOT NULL, + "raw_json" text, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_default_setup_assistants" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "profile_uuid" varchar(255) NOT NULL DEFAULT '', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "abm_token_id" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_mdm_default_setup_assistant_global_or_team_id_abm_token_id" UNIQUE ("global_or_team_id","abm_token_id") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_enrollment_profiles" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "token" varchar(36) DEFAULT NULL, + "type" varchar(10) NOT NULL DEFAULT 'automatic', + "dep_profile" jsonb DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_type" UNIQUE ("type"), + CONSTRAINT "mdm_apple_enrollment_profiles_idx_token" UNIQUE ("token") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_installers" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "name" varchar(255) NOT NULL DEFAULT '', + "size" bigint NOT NULL, + "manifest" text NOT NULL, + "installer" bytea, + "url_token" varchar(36) DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_setup_assistant_profiles" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "setup_assistant_id" int NOT NULL, + "abm_token_id" int NOT NULL, + "profile_uuid" varchar(255) NOT NULL DEFAULT '', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_mdm_apple_setup_assistant_profiles_asst_id_tok_id" UNIQUE ("setup_assistant_id","abm_token_id") +); + +CREATE TABLE IF NOT EXISTS "mdm_apple_setup_assistants" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "name" text NOT NULL, + "profile" jsonb NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_mdm_setup_assistant_global_or_team_id" UNIQUE ("global_or_team_id") +); + +CREATE TABLE IF NOT EXISTS "mdm_config_assets" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "name" varchar(256) NOT NULL DEFAULT '', + "value" bytea NOT NULL, + "deleted_at" timestamp NULL DEFAULT NULL, + "deletion_uuid" varchar(127) NOT NULL DEFAULT '', + "md5_checksum" bytea NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_mdm_config_assets_name_deletion_uuid" UNIQUE ("name","deletion_uuid") +); + +CREATE TABLE IF NOT EXISTS "mdm_configuration_profile_labels" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "apple_profile_uuid" varchar(37) DEFAULT NULL, + "windows_profile_uuid" varchar(37) DEFAULT NULL, + "label_name" varchar(255) NOT NULL, + "label_id" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "exclude" boolean NOT NULL DEFAULT FALSE, + "require_all" boolean NOT NULL DEFAULT FALSE, + "android_profile_uuid" varchar(37) DEFAULT NULL, + CONSTRAINT "ck_mdm_configuration_profile_labels_profile_uuid" CHECK ((((CASE WHEN "apple_profile_uuid" IS NULL THEN 0 ELSE 1 END) + (CASE WHEN "windows_profile_uuid" IS NULL THEN 0 ELSE 1 END) + (CASE WHEN "android_profile_uuid" IS NULL THEN 0 ELSE 1 END)) = 1)), + PRIMARY KEY ("id"), + CONSTRAINT "idx_mdm_configuration_profile_labels_apple_label_name" UNIQUE ("apple_profile_uuid","label_name"), + CONSTRAINT "idx_mdm_configuration_profile_labels_windows_label_name" UNIQUE ("windows_profile_uuid","label_name"), + CONSTRAINT "idx_mdm_configuration_profile_labels_android_label_name" UNIQUE ("android_profile_uuid","label_name") +); + +CREATE TABLE IF NOT EXISTS "mdm_configuration_profile_variables" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "apple_profile_uuid" varchar(37) DEFAULT NULL, + "windows_profile_uuid" varchar(37) DEFAULT NULL, + "fleet_variable_id" int NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "ck_mdm_configuration_profile_variables_apple_or_windows" CHECK ((("apple_profile_uuid" is null) <> ("windows_profile_uuid" is null))), + PRIMARY KEY ("id"), + CONSTRAINT "idx_mdm_configuration_profile_variables_apple_variable" UNIQUE ("apple_profile_uuid","fleet_variable_id"), + CONSTRAINT "idx_mdm_configuration_profile_variables_windows_label_name" UNIQUE ("windows_profile_uuid","fleet_variable_id") +); + +CREATE TABLE IF NOT EXISTS "mdm_declaration_labels" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "apple_declaration_uuid" varchar(37) NOT NULL DEFAULT '', + "label_name" varchar(255) NOT NULL, + "label_id" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp NULL DEFAULT NULL, + "exclude" boolean NOT NULL DEFAULT FALSE, + "require_all" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id"), + CONSTRAINT "idx_mdm_declaration_labels_label_name" UNIQUE ("apple_declaration_uuid","label_name") +); + +CREATE TABLE IF NOT EXISTS "mdm_delivery_status" ( + "status" varchar(20) NOT NULL, + PRIMARY KEY ("status") +); +INSERT INTO "mdm_delivery_status" ("status") VALUES ('failed'), ('verifying'), ('pending'), ('verified') + ON CONFLICT DO NOTHING; + +CREATE TABLE IF NOT EXISTS "mdm_idp_accounts" ( + "uuid" varchar(255) NOT NULL, + "username" varchar(255) NOT NULL, + "fullname" varchar(256) NOT NULL DEFAULT '', + "email" varchar(255) NOT NULL DEFAULT '', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("uuid"), + CONSTRAINT "unique_idp_email" UNIQUE ("email") +); + +CREATE TABLE IF NOT EXISTS "mdm_operation_types" ( + "operation_type" varchar(20) NOT NULL, + PRIMARY KEY ("operation_type") +); +INSERT INTO "mdm_operation_types" ("operation_type") VALUES ('install'), ('remove') + ON CONFLICT DO NOTHING; + +CREATE TABLE IF NOT EXISTS "mdm_windows_configuration_profiles" ( + "team_id" int NOT NULL DEFAULT '0', + "name" varchar(255) NOT NULL, + "syncml" bytea NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp NULL DEFAULT NULL, + "profile_uuid" varchar(37) NOT NULL DEFAULT '', + "checksum" bytea, + "secrets_updated_at" timestamp DEFAULT NULL, + PRIMARY KEY ("profile_uuid"), + CONSTRAINT "idx_mdm_windows_configuration_profiles_team_id_name" UNIQUE ("team_id","name") +); + +CREATE TABLE IF NOT EXISTS "mdm_windows_enrollments" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "mdm_device_id" varchar(255) NOT NULL, + "mdm_hardware_id" varchar(255) NOT NULL, + "device_state" varchar(255) NOT NULL, + "device_type" varchar(255) NOT NULL, + "device_name" varchar(255) NOT NULL, + "enroll_type" varchar(255) NOT NULL, + "enroll_user_id" varchar(255) NOT NULL, + "enroll_proto_version" varchar(255) NOT NULL, + "enroll_client_version" varchar(255) NOT NULL, + "not_in_oobe" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "host_uuid" varchar(255) NOT NULL DEFAULT '', + "credentials_hash" bytea DEFAULT NULL, + "credentials_acknowledged" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id"), + CONSTRAINT "mdm_windows_enrollments_idx_type" UNIQUE ("mdm_hardware_id") +); + +CREATE TABLE IF NOT EXISTS "microsoft_compliance_partner_host_statuses" ( + "host_id" int NOT NULL, + "device_id" varchar(64) NOT NULL, + "user_principal_name" varchar(255) NOT NULL, + "managed" boolean DEFAULT NULL, + "compliant" boolean DEFAULT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("host_id") +); + +CREATE TABLE IF NOT EXISTS "microsoft_compliance_partner_integrations" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "tenant_id" varchar(64) NOT NULL, + "proxy_server_secret" varchar(64) NOT NULL, + "setup_done" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_microsoft_compliance_partner_tenant_id" UNIQUE ("tenant_id") +); + +CREATE TABLE IF NOT EXISTS "migration_status_tables" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "version_id" bigint NOT NULL, + "is_applied" boolean NOT NULL, + "tstamp" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "mobile_device_management_solutions" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "name" varchar(100) NOT NULL, + "server_url" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_mobile_device_management_solutions_name" UNIQUE ("name","server_url") +); + +CREATE TABLE IF NOT EXISTS "munki_issues" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "name" varchar(255) NOT NULL, + "issue_type" varchar(10) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_munki_issues_name" UNIQUE ("name","issue_type") +); + +CREATE TABLE IF NOT EXISTS "nano_cert_auth_associations" ( + "id" varchar(255) NOT NULL, + "sha256" char(64) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "cert_not_valid_after" timestamp NULL DEFAULT NULL, + "renew_command_uuid" varchar(127) DEFAULT NULL, + CONSTRAINT "nano_cert_auth_associations_chk_1" CHECK (("id" <> '')), + CONSTRAINT "nano_cert_auth_associations_chk_2" CHECK (("sha256" <> '')), + PRIMARY KEY ("id","sha256") +); + +CREATE TABLE IF NOT EXISTS "nano_command_results" ( + "id" varchar(255) NOT NULL, + "command_uuid" varchar(127) NOT NULL, + "status" varchar(31) NOT NULL, + "result" text NOT NULL, + "not_now_at" timestamp NULL DEFAULT NULL, + "not_now_tally" int NOT NULL DEFAULT '0', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "nano_command_results_chk_1" CHECK (("status" <> '')), + PRIMARY KEY ("id","command_uuid") +); + +CREATE TABLE IF NOT EXISTS "nano_commands" ( + "command_uuid" varchar(127) NOT NULL, + "request_type" varchar(63) NOT NULL, + "command" text NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "subtype" text NOT NULL DEFAULT 'None', + CONSTRAINT "nano_commands_chk_1" CHECK (("command_uuid" <> '')), + CONSTRAINT "nano_commands_chk_2" CHECK (("request_type" <> '')), + PRIMARY KEY ("command_uuid") +); + +CREATE TABLE IF NOT EXISTS "nano_dep_names" ( + "name" varchar(255) NOT NULL, + "consumer_key" text, + "consumer_secret" text, + "access_token" text, + "access_secret" text, + "access_token_expiry" timestamp NULL DEFAULT NULL, + "config_base_url" varchar(255) DEFAULT NULL, + "tokenpki_cert_pem" text, + "tokenpki_key_pem" text, + "syncer_cursor" varchar(1024) DEFAULT NULL, + "syncer_cursor_at" timestamp NULL DEFAULT NULL, + "assigner_profile_uuid" text, + "assigner_profile_uuid_at" timestamp NULL DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "nano_dep_names_chk_1" CHECK ((("tokenpki_cert_pem" is null) or (substr("tokenpki_cert_pem",1,27) = '-----BEGIN CERTIFICATE-----'))), + CONSTRAINT "nano_dep_names_chk_2" CHECK ((("tokenpki_key_pem" is null) or (substr("tokenpki_key_pem",1,5) = '-----'))), + PRIMARY KEY ("name") +); + +CREATE TABLE IF NOT EXISTS "nano_devices" ( + "id" varchar(255) NOT NULL, + "identity_cert" text, + "serial_number" varchar(127) DEFAULT NULL, + "unlock_token" bytea, + "unlock_token_at" timestamp NULL DEFAULT NULL, + "authenticate" text NOT NULL, + "authenticate_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "token_update" text, + "token_update_at" timestamp NULL DEFAULT NULL, + "bootstrap_token_b64" text, + "bootstrap_token_at" timestamp NULL DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "platform" varchar(255) NOT NULL DEFAULT '', + "enroll_team_id" int DEFAULT NULL, + CONSTRAINT "nano_devices_chk_1" CHECK ((("identity_cert" is null) or (substr("identity_cert",1,27) = '-----BEGIN CERTIFICATE-----'))), + CONSTRAINT "nano_devices_chk_2" CHECK ((("serial_number" is null) or ("serial_number" <> ''))), + CONSTRAINT "nano_devices_chk_3" CHECK ((("unlock_token" is null) or (length("unlock_token") > 0))), + CONSTRAINT "nano_devices_chk_4" CHECK (("authenticate" <> '')), + CONSTRAINT "nano_devices_chk_5" CHECK ((("token_update" is null) or ("token_update" <> ''))), + CONSTRAINT "nano_devices_chk_6" CHECK ((("bootstrap_token_b64" is null) or ("bootstrap_token_b64" <> ''))), + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "nano_enrollment_queue" ( + "id" varchar(255) NOT NULL, + "command_uuid" varchar(127) NOT NULL, + "active" boolean NOT NULL DEFAULT TRUE, + "priority" smallint NOT NULL DEFAULT '0', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id","command_uuid") +); + +CREATE TABLE IF NOT EXISTS "nano_enrollments" ( + "id" varchar(255) NOT NULL, + "device_id" varchar(255) NOT NULL, + "user_id" varchar(255) DEFAULT NULL, + "type" varchar(31) NOT NULL, + "topic" varchar(255) NOT NULL, + "push_magic" varchar(127) NOT NULL, + "token_hex" varchar(255) NOT NULL, + "enabled" boolean NOT NULL DEFAULT TRUE, + "token_update_tally" int NOT NULL DEFAULT '1', + "last_seen_at" timestamp NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "enrolled_from_migration" smallint NOT NULL DEFAULT '0', + CONSTRAINT "nano_enrollments_chk_1" CHECK (("id" <> '')), + CONSTRAINT "nano_enrollments_chk_2" CHECK (("type" <> '')), + CONSTRAINT "nano_enrollments_chk_3" CHECK (("topic" <> '')), + CONSTRAINT "nano_enrollments_chk_4" CHECK (("push_magic" <> '')), + CONSTRAINT "nano_enrollments_chk_5" CHECK (("token_hex" <> '')), + PRIMARY KEY ("id"), + CONSTRAINT "user_id" UNIQUE ("user_id") +); + +CREATE TABLE IF NOT EXISTS "nano_push_certs" ( + "topic" varchar(255) NOT NULL, + "cert_pem" text NOT NULL, + "key_pem" text NOT NULL, + "stale_token" int NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "nano_push_certs_chk_1" CHECK (("topic" <> '')), + CONSTRAINT "nano_push_certs_chk_2" CHECK ((substr("cert_pem",1,27) = '-----BEGIN CERTIFICATE-----')), + CONSTRAINT "nano_push_certs_chk_3" CHECK ((substr("key_pem",1,5) = '-----')), + PRIMARY KEY ("topic") +); + +CREATE TABLE IF NOT EXISTS "nano_users" ( + "id" varchar(255) NOT NULL, + "device_id" varchar(255) NOT NULL, + "user_short_name" varchar(255) DEFAULT NULL, + "user_long_name" varchar(255) DEFAULT NULL, + "token_update" text, + "token_update_at" timestamp NULL DEFAULT NULL, + "user_authenticate" text, + "user_authenticate_at" timestamp NULL DEFAULT NULL, + "user_authenticate_digest" text, + "user_authenticate_digest_at" timestamp NULL DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "nano_users_chk_1" CHECK ((("user_short_name" is null) or ("user_short_name" <> ''))), + CONSTRAINT "nano_users_chk_2" CHECK ((("user_long_name" is null) or ("user_long_name" <> ''))), + CONSTRAINT "nano_users_chk_3" CHECK ((("token_update" is null) or ("token_update" <> ''))), + CONSTRAINT "nano_users_chk_4" CHECK ((("user_authenticate" is null) or ("user_authenticate" <> ''))), + CONSTRAINT "nano_users_chk_5" CHECK ((("user_authenticate_digest" is null) or ("user_authenticate_digest" <> ''))), + PRIMARY KEY ("id","device_id"), + CONSTRAINT "idx_unique_id" UNIQUE ("id") +); + +CREATE TABLE IF NOT EXISTS "network_interfaces" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_id" int NOT NULL, + "mac" varchar(255) NOT NULL DEFAULT '', + "ip_address" varchar(255) NOT NULL DEFAULT '', + "broadcast" varchar(255) NOT NULL DEFAULT '', + "ibytes" bigint NOT NULL DEFAULT '0', + "interface" varchar(255) NOT NULL DEFAULT '', + "ipackets" bigint NOT NULL DEFAULT '0', + "last_change" bigint NOT NULL DEFAULT '0', + "mask" varchar(255) NOT NULL DEFAULT '', + "metric" int NOT NULL DEFAULT '0', + "mtu" int NOT NULL DEFAULT '0', + "obytes" bigint NOT NULL DEFAULT '0', + "ierrors" bigint NOT NULL DEFAULT '0', + "oerrors" bigint NOT NULL DEFAULT '0', + "opackets" bigint NOT NULL DEFAULT '0', + "point_to_point" varchar(255) NOT NULL DEFAULT '', + "type" int NOT NULL DEFAULT '0', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_network_interfaces_unique_ip_host_intf" UNIQUE ("ip_address","host_id","interface") +); + +CREATE TABLE IF NOT EXISTS "operating_system_version_vulnerabilities" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "os_version_id" int NOT NULL, + "cve" varchar(255) NOT NULL, + "team_id" int DEFAULT NULL, + "source" smallint DEFAULT '0', + "resolved_in_version" varchar(255) DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id") +); +CREATE UNIQUE INDEX IF NOT EXISTS "idx_os_version_vulnerabilities_unq_os_version_team_cve" ON "operating_system_version_vulnerabilities" ((COALESCE("team_id",-1)),"os_version_id","cve"); + +CREATE TABLE IF NOT EXISTS "operating_system_vulnerabilities" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "operating_system_id" int NOT NULL, + "cve" varchar(255) NOT NULL, + "source" smallint DEFAULT '0', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "resolved_in_version" varchar(255) DEFAULT NULL, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_os_vulnerabilities_unq_os_id_cve" UNIQUE ("operating_system_id","cve") +); + +CREATE TABLE IF NOT EXISTS "operating_systems" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "name" varchar(255) NOT NULL, + "version" varchar(150) NOT NULL, + "arch" varchar(150) NOT NULL, + "kernel_version" varchar(150) NOT NULL, + "platform" varchar(50) NOT NULL, + "display_version" varchar(10) NOT NULL DEFAULT '', + "installation_type" varchar(20) NOT NULL DEFAULT '', + "os_version_id" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_os" UNIQUE ("name","version","arch","kernel_version","platform","display_version","installation_type") +); + +CREATE TABLE IF NOT EXISTS "osquery_options" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "override_type" int NOT NULL, + "override_identifier" varchar(255) NOT NULL DEFAULT '', + "options" jsonb NOT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "pack_targets" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "pack_id" int DEFAULT NULL, + "type" int DEFAULT NULL, + "target_id" int NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "constraint_pack_target_unique" UNIQUE ("pack_id","target_id","type") +); + +CREATE TABLE IF NOT EXISTS "packs" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "disabled" boolean NOT NULL DEFAULT FALSE, + "name" varchar(255) NOT NULL, + "description" varchar(255) NOT NULL DEFAULT '', + "platform" varchar(255) NOT NULL DEFAULT '', + "pack_type" varchar(255) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_pack_unique_name" UNIQUE ("name") +); + +CREATE TABLE IF NOT EXISTS "password_reset_requests" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "expires_at" timestamp NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "user_id" int NOT NULL, + "token" varchar(1024) NOT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "policies" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "team_id" int DEFAULT NULL, + "resolution" text, + "name" varchar(255) NOT NULL, + "query" text NOT NULL, + "description" text NOT NULL, + "author_id" int DEFAULT NULL, + "platforms" varchar(255) NOT NULL DEFAULT '', + "critical" boolean NOT NULL DEFAULT FALSE, + "checksum" bytea NOT NULL, + "calendar_events_enabled" boolean NOT NULL DEFAULT FALSE, + "software_installer_id" int DEFAULT NULL, + "script_id" int DEFAULT NULL, + "vpp_apps_teams_id" int DEFAULT NULL, + "conditional_access_enabled" boolean NOT NULL DEFAULT FALSE, + "conditional_access_bypass_enabled" boolean NOT NULL DEFAULT TRUE, + "type" varchar(255) NOT NULL DEFAULT 'dynamic', + "patch_software_title_id" int DEFAULT NULL, + "needs_full_membership_cleanup" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("id"), + CONSTRAINT "idx_policies_checksum" UNIQUE ("checksum") +); + +CREATE TABLE IF NOT EXISTS "policy_automation_iterations" ( + "policy_id" int NOT NULL, + "iteration" int NOT NULL, + PRIMARY KEY ("policy_id") +); + +CREATE TABLE IF NOT EXISTS "policy_labels" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "policy_id" int NOT NULL, + "label_id" int NOT NULL, + "exclude" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_policy_labels_policy_label" UNIQUE ("policy_id","label_id") +); + +CREATE TABLE IF NOT EXISTS "policy_membership" ( + "policy_id" int NOT NULL, + "host_id" int NOT NULL, + "passes" boolean DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "automation_iteration" int DEFAULT NULL, + PRIMARY KEY ("policy_id","host_id") +); + +CREATE TABLE IF NOT EXISTS "policy_stats" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "policy_id" int NOT NULL, + "inherited_team_id" int DEFAULT NULL, + "passing_host_count" integer NOT NULL DEFAULT '0', + "failing_host_count" integer NOT NULL DEFAULT '0', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "inherited_team_id_char" text GENERATED ALWAYS AS (CASE WHEN inherited_team_id IS NULL THEN 'global' ELSE inherited_team_id::text END) STORED, + PRIMARY KEY ("id"), + CONSTRAINT "policy_id" UNIQUE ("policy_id","inherited_team_id_char") +); + +CREATE TABLE IF NOT EXISTS "queries" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "saved" boolean NOT NULL DEFAULT FALSE, + "name" varchar(255) NOT NULL, + "description" text NOT NULL, + "query" text NOT NULL, + "author_id" int DEFAULT NULL, + "observer_can_run" boolean NOT NULL DEFAULT FALSE, + "team_id" int DEFAULT NULL, + "team_id_char" char(10) NOT NULL DEFAULT '', + "platform" varchar(255) NOT NULL DEFAULT '', + "min_osquery_version" varchar(255) NOT NULL DEFAULT '', + "schedule_interval" int NOT NULL DEFAULT '0', + "automations_enabled" boolean NOT NULL DEFAULT FALSE, + "logging_type" varchar(255) NOT NULL DEFAULT 'snapshot', + "discard_data" boolean NOT NULL DEFAULT TRUE, + "is_scheduled" boolean GENERATED ALWAYS AS (schedule_interval > 0) STORED, + PRIMARY KEY ("id"), + CONSTRAINT "idx_team_id_name_unq" UNIQUE ("team_id_char","name"), + CONSTRAINT "idx_name_team_id_unq" UNIQUE ("name","team_id_char") +); + +CREATE TABLE IF NOT EXISTS "query_labels" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "query_id" int NOT NULL, + "label_id" int NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_query_labels_query_label" UNIQUE ("query_id","label_id") +); + +CREATE TABLE IF NOT EXISTS "query_results" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "query_id" int NOT NULL, + "host_id" int NOT NULL, + "osquery_version" varchar(50) DEFAULT NULL, + "error" text, + "last_fetched" timestamp NOT NULL, + "data" jsonb DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_query_results_query_host" UNIQUE ("query_id","host_id") +); + +CREATE TABLE IF NOT EXISTS "scep_certificates" ( + "serial" bigint NOT NULL, + "name" varchar(1024) DEFAULT NULL, + "not_valid_before" timestamp NOT NULL, + "not_valid_after" timestamp NOT NULL, + "certificate_pem" text NOT NULL, + "revoked" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + CONSTRAINT "scep_certificates_chk_1" CHECK ((substr("certificate_pem",1,27) = '-----BEGIN CERTIFICATE-----')), + CONSTRAINT "scep_certificates_chk_2" CHECK ((("name" is null) or ("name" <> ''))), + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "scep_serials" ( + "serial" bigint GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "scheduled_queries" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "pack_id" int DEFAULT NULL, + "query_id" int DEFAULT NULL, + "interval" int DEFAULT NULL, + "snapshot" boolean DEFAULT NULL, + "removed" boolean DEFAULT NULL, + "platform" varchar(255) DEFAULT '', + "version" varchar(255) DEFAULT '', + "shard" int DEFAULT NULL, + "query_name" varchar(255) NOT NULL, + "name" varchar(255) NOT NULL, + "description" varchar(1023) DEFAULT '', + "denylist" boolean DEFAULT NULL, + "team_id_char" char(10) NOT NULL DEFAULT '', + PRIMARY KEY ("id"), + CONSTRAINT "unique_names_in_packs" UNIQUE ("name","pack_id") +); + +CREATE TABLE IF NOT EXISTS "scheduled_query_stats" ( + "host_id" int NOT NULL, + "scheduled_query_id" int NOT NULL, + "average_memory" bigint NOT NULL DEFAULT 0, + "denylisted" boolean DEFAULT NULL, + "executions" bigint NOT NULL DEFAULT 0, + "schedule_interval" int DEFAULT NULL, + "last_executed" timestamp NULL DEFAULT NULL, + "output_size" bigint NOT NULL DEFAULT 0, + "system_time" bigint NOT NULL DEFAULT 0, + "user_time" bigint NOT NULL DEFAULT 0, + "wall_time" bigint NOT NULL DEFAULT 0, + "query_type" smallint NOT NULL DEFAULT '0', + PRIMARY KEY ("host_id","scheduled_query_id","query_type") +); + +CREATE TABLE IF NOT EXISTS "scim_groups" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "external_id" varchar(255) DEFAULT NULL, + "display_name" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_scim_groups_display_name" UNIQUE ("display_name") +); + +CREATE TABLE IF NOT EXISTS "scim_last_request" ( + "id" smallint NOT NULL DEFAULT '1', + "status" varchar(31) NOT NULL, + "details" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "scim_user_emails" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "scim_user_id" int NOT NULL, + "email" varchar(255) NOT NULL, + "primary" boolean DEFAULT NULL, + "type" varchar(31) DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "scim_user_group" ( + "scim_user_id" int NOT NULL, + "group_id" int NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("scim_user_id","group_id") +); + +CREATE TABLE IF NOT EXISTS "scim_users" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "external_id" varchar(255) DEFAULT NULL, + "user_name" varchar(255) NOT NULL, + "given_name" varchar(255) DEFAULT NULL, + "family_name" varchar(255) DEFAULT NULL, + "active" boolean DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "department" varchar(255) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_scim_users_user_name" UNIQUE ("user_name") +); + +CREATE TABLE IF NOT EXISTS "script_contents" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "md5_checksum" bytea NOT NULL, + "contents" text NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_script_contents_md5_checksum" UNIQUE ("md5_checksum") +); + +CREATE TABLE IF NOT EXISTS "script_upcoming_activities" ( + "upcoming_activity_id" bigint NOT NULL, + "script_id" int DEFAULT NULL, + "script_content_id" int DEFAULT NULL, + "policy_id" int DEFAULT NULL, + "setup_experience_script_id" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("upcoming_activity_id") +); + +CREATE TABLE IF NOT EXISTS "scripts" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "name" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "script_content_id" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_scripts_global_or_team_id_name" UNIQUE ("global_or_team_id","name"), + CONSTRAINT "idx_scripts_team_name" UNIQUE ("team_id","name") +); + +CREATE TABLE IF NOT EXISTS "secret_variables" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "name" varchar(255) NOT NULL, + "value" bytea NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_secret_variables_name" UNIQUE ("name") +); + +CREATE TABLE IF NOT EXISTS "sessions" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "accessed_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "user_id" int NOT NULL, + "key" varchar(255) NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_session_unique_key" UNIQUE ("key") +); + +CREATE TABLE IF NOT EXISTS "setup_experience_scripts" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "name" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "script_content_id" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_setup_experience_scripts_global_or_team_id" UNIQUE ("global_or_team_id") +); + +CREATE TABLE IF NOT EXISTS "setup_experience_status_results" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_uuid" varchar(255) NOT NULL, + "name" varchar(255) NOT NULL, + "status" text NOT NULL, + "software_installer_id" int DEFAULT NULL, + "host_software_installs_execution_id" varchar(255) DEFAULT NULL, + "vpp_app_team_id" int DEFAULT NULL, + "nano_command_uuid" varchar(255) DEFAULT NULL, + "setup_experience_script_id" int DEFAULT NULL, + "script_execution_id" varchar(255) DEFAULT NULL, + "error" varchar(255) DEFAULT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "software" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "name" varchar(255) NOT NULL, + "version" varchar(255) NOT NULL DEFAULT '', + "source" varchar(64) NOT NULL, + "bundle_identifier" varchar(255) DEFAULT '', + "release" varchar(64) NOT NULL DEFAULT '', + "vendor_old" varchar(32) NOT NULL DEFAULT '', + "arch" varchar(16) NOT NULL DEFAULT '', + "vendor" varchar(114) NOT NULL DEFAULT '', + "extension_for" varchar(255) NOT NULL DEFAULT '', + "extension_id" varchar(255) NOT NULL DEFAULT '', + "title_id" int DEFAULT NULL, + "checksum" bytea NOT NULL, + "name_source" text NOT NULL DEFAULT 'basic', + "application_id" varchar(255) DEFAULT NULL, + "upgrade_code" char(38) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_software_checksum" UNIQUE ("checksum") +); + +CREATE TABLE IF NOT EXISTS "software_categories" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "name" varchar(63) NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_software_categories_name" UNIQUE ("name") +); + +CREATE TABLE IF NOT EXISTS "software_cpe" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "software_id" bigint DEFAULT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "cpe" varchar(255) NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "unq_software_id" UNIQUE ("software_id") +); + +CREATE TABLE IF NOT EXISTS "software_cve" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "cve" varchar(255) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "source" int DEFAULT '0', + "software_id" bigint DEFAULT NULL, + "resolved_in_version" varchar(255) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "unq_software_id_cve" UNIQUE ("software_id","cve") +); + +CREATE TABLE IF NOT EXISTS "software_host_counts" ( + "software_id" bigint NOT NULL, + "hosts_count" int NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "team_id" int NOT NULL DEFAULT '0', + "global_stats" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("software_id","team_id","global_stats") +); + +CREATE TABLE IF NOT EXISTS "software_install_upcoming_activities" ( + "upcoming_activity_id" bigint NOT NULL, + "software_installer_id" int DEFAULT NULL, + "policy_id" int DEFAULT NULL, + "software_title_id" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("upcoming_activity_id") +); + +CREATE TABLE IF NOT EXISTS "software_installer_labels" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "software_installer_id" int NOT NULL, + "label_id" int NOT NULL, + "exclude" boolean NOT NULL DEFAULT FALSE, + "require_all" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_software_installer_labels_software_installer_id_label_id" UNIQUE ("software_installer_id","label_id") +); + +CREATE TABLE IF NOT EXISTS "software_installer_software_categories" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "software_category_id" int NOT NULL, + "software_installer_id" int NOT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_software_installer_id_software_category_id" UNIQUE ("software_installer_id","software_category_id") +); + +CREATE TABLE IF NOT EXISTS "software_installers" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "title_id" int DEFAULT NULL, + "filename" varchar(255) NOT NULL, + "version" varchar(255) NOT NULL, + "platform" varchar(255) NOT NULL, + "pre_install_query" text, + "install_script_content_id" int NOT NULL, + "post_install_script_content_id" int DEFAULT NULL, + "storage_id" varchar(64) NOT NULL, + "uploaded_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "self_service" boolean NOT NULL DEFAULT FALSE, + "user_id" int DEFAULT NULL, + "user_name" varchar(255) NOT NULL DEFAULT '', + "user_email" varchar(255) NOT NULL DEFAULT '', + "url" varchar(4095) NOT NULL DEFAULT '', + "package_ids" text NOT NULL, + "extension" varchar(32) NOT NULL DEFAULT '', + "uninstall_script_content_id" int NOT NULL, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "fleet_maintained_app_id" int DEFAULT NULL, + "install_during_setup" boolean NOT NULL DEFAULT FALSE, + "is_active" boolean NOT NULL DEFAULT TRUE, + "upgrade_code" varchar(48) NOT NULL DEFAULT '', + "patch_query" text NOT NULL DEFAULT '', + PRIMARY KEY ("id"), + CONSTRAINT "idx_software_installers_team_id_title_id" UNIQUE ("global_or_team_id","title_id") +); + +CREATE TABLE IF NOT EXISTS "software_title_display_names" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "team_id" int NOT NULL, + "software_title_id" int NOT NULL, + "display_name" varchar(255) NOT NULL DEFAULT '', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_team_id_title_id" UNIQUE ("team_id","software_title_id") +); + +CREATE TABLE IF NOT EXISTS "software_title_icons" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "team_id" int NOT NULL, + "software_title_id" int NOT NULL, + "storage_id" varchar(64) NOT NULL, + "filename" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_team_id_title_id_storage_id" UNIQUE ("team_id","software_title_id") +); + +CREATE TABLE IF NOT EXISTS "software_titles" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "name" varchar(255) NOT NULL, + "source" varchar(64) NOT NULL, + "extension_for" varchar(255) NOT NULL DEFAULT '', + "bundle_identifier" varchar(255) DEFAULT NULL, + "additional_identifier" text, + "is_kernel" boolean NOT NULL DEFAULT FALSE, + "application_id" varchar(255) DEFAULT NULL, + "unique_identifier" text, + "upgrade_code" char(38) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_software_titles_bundle_identifier" UNIQUE ("bundle_identifier","additional_identifier"), + CONSTRAINT "idx_unique_sw_titles" UNIQUE ("unique_identifier","source","extension_for") +); + +CREATE TABLE IF NOT EXISTS "software_titles_host_counts" ( + "software_title_id" int NOT NULL, + "hosts_count" int NOT NULL, + "team_id" int NOT NULL DEFAULT '0', + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + "global_stats" boolean NOT NULL DEFAULT FALSE, + PRIMARY KEY ("software_title_id","team_id","global_stats") +); + +CREATE TABLE IF NOT EXISTS "software_update_schedules" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "team_id" int NOT NULL, + "title_id" int NOT NULL, + "enabled" boolean NOT NULL DEFAULT FALSE, + "start_time" char(5) NOT NULL, + "end_time" char(5) NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_team_title" UNIQUE ("team_id","title_id") +); + +CREATE TABLE IF NOT EXISTS "statistics" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "anonymous_identifier" varchar(255) NOT NULL, + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "teams" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "name" varchar(255) NOT NULL, + "description" varchar(1023) NOT NULL DEFAULT '', + "config" jsonb DEFAULT NULL, + "name_bin" text, + "filename" varchar(255) DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_teams_filename" UNIQUE ("filename"), + CONSTRAINT "idx_name_bin" UNIQUE ("name_bin") +); + +CREATE TABLE IF NOT EXISTS "upcoming_activities" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "host_id" int NOT NULL, + "priority" int NOT NULL DEFAULT '0', + "user_id" int DEFAULT NULL, + "fleet_initiated" boolean NOT NULL DEFAULT FALSE, + "activity_type" text NOT NULL, + "execution_id" varchar(255) NOT NULL, + "payload" jsonb NOT NULL, + "activated_at" timestamp DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_upcoming_activities_execution_id" UNIQUE ("execution_id") +); + +CREATE TABLE IF NOT EXISTS "user_teams" ( + "user_id" int NOT NULL, + "team_id" int NOT NULL, + "role" varchar(64) NOT NULL, + PRIMARY KEY ("user_id","team_id") +); + +CREATE TABLE IF NOT EXISTS "users" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "password" bytea NOT NULL, + "salt" varchar(255) NOT NULL, + "name" varchar(255) NOT NULL DEFAULT '', + "email" varchar(255) NOT NULL, + "admin_forced_password_reset" boolean NOT NULL DEFAULT FALSE, + "gravatar_url" varchar(255) NOT NULL DEFAULT '', + "position" varchar(255) NOT NULL DEFAULT '', + "sso_enabled" boolean NOT NULL DEFAULT FALSE, + "global_role" varchar(64) DEFAULT NULL, + "api_only" boolean NOT NULL DEFAULT FALSE, + "mfa_enabled" boolean NOT NULL DEFAULT FALSE, + "settings" jsonb NOT NULL DEFAULT '{}'::jsonb, + "invite_id" int DEFAULT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_user_unique_email" UNIQUE ("email"), + CONSTRAINT "invite_id" UNIQUE ("invite_id") +); + +CREATE TABLE IF NOT EXISTS "users_deleted" ( + "id" int NOT NULL, + "name" varchar(255) NOT NULL DEFAULT '', + "email" varchar(255) NOT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "verification_tokens" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "user_id" int NOT NULL, + "token" varchar(255) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "token" UNIQUE ("token") +); + +CREATE TABLE IF NOT EXISTS "vpp_app_team_labels" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "vpp_app_team_id" int NOT NULL, + "label_id" int NOT NULL, + "exclude" boolean NOT NULL DEFAULT FALSE, + "require_all" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_vpp_app_team_labels_vpp_app_team_id_label_id" UNIQUE ("vpp_app_team_id","label_id") +); + +CREATE TABLE IF NOT EXISTS "vpp_app_team_software_categories" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "software_category_id" int NOT NULL, + "vpp_app_team_id" int NOT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_vpp_app_team_id_software_category_id" UNIQUE ("vpp_app_team_id","software_category_id") +); + +CREATE TABLE IF NOT EXISTS "vpp_app_upcoming_activities" ( + "upcoming_activity_id" bigint NOT NULL, + "adam_id" varchar(255) NOT NULL, + "platform" varchar(10) NOT NULL, + "vpp_token_id" int DEFAULT NULL, + "policy_id" int DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("upcoming_activity_id") +); + +CREATE TABLE IF NOT EXISTS "vpp_apps" ( + "adam_id" varchar(255) NOT NULL, + "title_id" int DEFAULT NULL, + "bundle_identifier" varchar(255) NOT NULL DEFAULT '', + "icon_url" varchar(255) NOT NULL DEFAULT '', + "name" varchar(255) NOT NULL DEFAULT '', + "latest_version" varchar(255) NOT NULL DEFAULT '', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "platform" varchar(10) NOT NULL, + PRIMARY KEY ("adam_id","platform") +); + +CREATE TABLE IF NOT EXISTS "vpp_apps_teams" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "adam_id" varchar(255) NOT NULL, + "team_id" int DEFAULT NULL, + "global_or_team_id" int NOT NULL DEFAULT '0', + "platform" varchar(10) NOT NULL, + "self_service" boolean NOT NULL DEFAULT FALSE, + "vpp_token_id" int DEFAULT NULL, + "install_during_setup" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_global_or_team_id_adam_id" UNIQUE ("global_or_team_id","adam_id","platform") +); + +CREATE TABLE IF NOT EXISTS "vpp_token_teams" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "vpp_token_id" int NOT NULL, + "team_id" int DEFAULT NULL, + "null_team_type" text DEFAULT 'none', + PRIMARY KEY ("id"), + CONSTRAINT "idx_vpp_token_teams_team_id" UNIQUE ("team_id") +); + +CREATE TABLE IF NOT EXISTS "vpp_tokens" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "organization_name" varchar(255) NOT NULL, + "location" varchar(255) NOT NULL, + "renew_at" timestamp NOT NULL, + "token" bytea NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id"), + CONSTRAINT "idx_vpp_tokens_location" UNIQUE ("location") +); + +CREATE TABLE IF NOT EXISTS "vulnerability_host_counts" ( + "cve" varchar(20) NOT NULL, + "team_id" int NOT NULL DEFAULT '0', + "host_count" int NOT NULL DEFAULT '0', + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + "global_stats" boolean NOT NULL DEFAULT FALSE, + CONSTRAINT "cve_team_id_global_stats" UNIQUE ("cve","team_id","global_stats") +); + +CREATE TABLE IF NOT EXISTS "windows_mdm_command_queue" ( + "enrollment_id" int NOT NULL, + "command_uuid" varchar(127) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("enrollment_id","command_uuid") +); + +CREATE TABLE IF NOT EXISTS "windows_mdm_command_results" ( + "enrollment_id" int NOT NULL, + "command_uuid" varchar(127) NOT NULL, + "raw_result" text NOT NULL, + "response_id" int NOT NULL, + "status_code" varchar(31) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("enrollment_id","command_uuid") +); + +CREATE TABLE IF NOT EXISTS "windows_mdm_commands" ( + "command_uuid" varchar(127) NOT NULL, + "raw_command" text NOT NULL, + "target_loc_uri" varchar(255) NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("command_uuid") +); + +CREATE TABLE IF NOT EXISTS "windows_mdm_responses" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "enrollment_id" int NOT NULL, + "raw_response" text NOT NULL, + "created_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "windows_updates" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "host_id" int NOT NULL, + "date_epoch" int NOT NULL, + "kb_id" int NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_unique_windows_updates" UNIQUE ("host_id","kb_id") +); + +CREATE TABLE IF NOT EXISTS "wstep_cert_auth_associations" ( + "id" varchar(255) NOT NULL, + "sha256" char(64) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("id","sha256") +); + +CREATE TABLE IF NOT EXISTS "wstep_certificates" ( + "serial" bigint NOT NULL, + "name" varchar(1024) NOT NULL, + "not_valid_before" timestamp NOT NULL, + "not_valid_after" timestamp NOT NULL, + "certificate_pem" text NOT NULL, + "revoked" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NULL DEFAULT CURRENT_TIMESTAMP , + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "wstep_serials" ( + "serial" bigint GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("serial") +); + +CREATE TABLE IF NOT EXISTS "yara_rules" ( + "id" int NOT NULL GENERATED BY DEFAULT AS IDENTITY, + "name" varchar(255) NOT NULL, + "contents" text NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "idx_yara_rules_name" UNIQUE ("name") +); + + +CREATE TABLE IF NOT EXISTS "migration_status_tables" ( + id serial NOT NULL PRIMARY KEY, version_id bigint NOT NULL, + is_applied boolean NOT NULL, tstamp timestamp DEFAULT now()); +CREATE TABLE IF NOT EXISTS "migration_status_data" ( + id serial NOT NULL PRIMARY KEY, version_id bigint NOT NULL, + is_applied boolean NOT NULL, tstamp timestamp DEFAULT now()); + +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118193812, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118211713, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212436, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212515, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212528, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212538, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212549, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212557, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212604, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212613, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212621, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212630, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212641, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212649, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212656, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161118212758, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161128234849, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20161230162221, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170104113816, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170105151732, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170108191242, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170109094020, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170109130438, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170110202752, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170111133013, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170117025759, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170118191001, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170119234632, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170124230432, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170127014618, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170131232841, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170223094154, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170306075207, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170309100733, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170331111922, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170502143928, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170504130602, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170509132100, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170519105647, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170519105648, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170831234300, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170831234301, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20170831234303, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20171116163618, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20171219164727, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20180620164811, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20180620175054, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20180620175055, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20191010101639, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20191010155147, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20191220130734, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20200311140000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20200405120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20200407120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20200420120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20200504120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20200512120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20200707120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20201011162341, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20201021104586, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20201102112520, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20201208121729, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20201215091637, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210119174155, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210326182902, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210421112652, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210506095025, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210513115729, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210526113559, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000001, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000002, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000003, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000004, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000005, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000006, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000007, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210601000008, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210606151329, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210616163757, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210617174723, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210622160235, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210623100031, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210623133615, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210708143152, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210709124443, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210712155608, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210714102108, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210719153709, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210721171531, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210723135713, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210802135933, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210806112844, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210810095603, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210811150223, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210818151827, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210818151828, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210818182258, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210819131107, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210819143446, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210903132338, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210915144307, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210920155130, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210927143115, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20210927143116, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211013133706, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211013133707, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211102135149, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211109121546, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211110163320, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211116184029, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211116184030, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211202092042, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211202181033, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211207161856, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211216131203, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20211221110132, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220107155700, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220125105650, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220201084510, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220208144830, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220208144831, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220208144831, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220215152203, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220215152203, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220223113157, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220223113157, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220307104655, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220309133956, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220309133956, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220316155700, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220323152301, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220323152301, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220330100659, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220330100659, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220404091216, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220404091216, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220419140750, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220419140750, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220428140039, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220428140039, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220503134048, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220503134048, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220524102918, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220524102918, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220526123327, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220526123327, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220526123328, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220526123329, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220608113128, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220608113128, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220627104817, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220627104817, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220704101843, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220704101843, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220708095046, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220708095046, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220713091130, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220713091130, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220802135510, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220802135510, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220818101352, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220818101352, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220822161445, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220822161445, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220831100036, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220831100151, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220831100151, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220908181826, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220908181826, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220914154915, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220915165115, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220915165116, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220915165116, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220928100158, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20220928100158, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221014084130, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221014084130, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221027085019, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221101103952, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221101103952, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221104144401, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221109100749, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221109100749, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221115104546, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221115104546, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221130114928, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221205112142, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221205112142, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221216115820, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221220195934, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221220195934, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221220195935, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221223174807, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221223174807, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221227163855, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221227163855, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20221227163856, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230202224725, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230202224725, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230206163608, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230206163608, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230214131519, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230214131519, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230303135738, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230303135738, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230313135301, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230313135301, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230313141819, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230313141819, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230315104937, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230315104937, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230317173844, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230317173844, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230320133602, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230320133602, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230330100011, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230330100011, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230330134823, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230330134823, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230405232025, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230405232025, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230408084104, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230408084104, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230411102858, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230411102858, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230421155932, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230421155932, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230425082126, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230425082126, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230425105727, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230425105727, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230501154913, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230501154913, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230503101418, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230503101418, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230515144206, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230515144206, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230517140952, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230517152807, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230517152807, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230518114155, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230518114155, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230520153236, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230520153236, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230525151159, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230530122103, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230530122103, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230602111827, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230602111827, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230608103123, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230608103123, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230629140529, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230629140530, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230629140530, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230711144622, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230711144622, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230721135421, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230721161508, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230721161508, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230726115701, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230807100822, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230814150442, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230814150442, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230823122728, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230823122728, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230906152143, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230906152143, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230911163618, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230911163618, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230912101759, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230912101759, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230915101341, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230915101341, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230918132351, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20230918132351, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231004144339, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231004144339, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231009094541, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231009094542, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231009094542, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231009094543, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231009094543, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231009094544, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231009094544, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231016091915, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231016091915, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231024174135, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231024174135, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231025120016, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231025120016, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231025160156, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231025160156, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231031165350, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231106144110, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231106144110, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231107130934, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231107130934, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231109115838, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231109115838, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231121054530, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231121054530, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231122101320, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231122101320, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231130132828, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231130132828, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231130132931, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231130132931, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231204155427, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231204155427, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231206142340, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231206142340, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231207102320, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231207102320, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231207102321, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231207102321, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231207133731, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231207133731, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231212094238, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231212094238, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231212095734, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231212161121, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231212161121, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231215122713, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231215122713, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231219143041, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231219143041, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231224070653, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20231224070653, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240110134315, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240110134315, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240119091637, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240119091637, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240126020642, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240126020642, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240126020643, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240126020643, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240129162819, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240129162819, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240130115133, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240130115133, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240131083822, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240131083822, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240205095928, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240205095928, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240205121956, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240209110212, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240209110212, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240212111533, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240221112844, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240221112844, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240222073518, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240222073518, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240222135115, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240222135115, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240226082255, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240226082255, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240228082706, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240228082706, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240301173035, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240301173035, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240302111134, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240302111134, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240312103753, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240313143416, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240314085226, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240314085226, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240314151747, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240314151747, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240320145650, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240320145650, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240327115530, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240327115617, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240408085837, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240408085837, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240415104633, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240415104633, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240430111727, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240430111727, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240515200020, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240521143023, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240521143023, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240521143024, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240521143024, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240601174138, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240601174138, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240607133721, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240607133721, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240612150059, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240612150059, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240613162201, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240613172616, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240613172616, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240618142419, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240618142419, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240625093543, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240625093543, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240626195531, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240626195531, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240702123921, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240703154849, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240703154849, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240707134035, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240707134035, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240707134036, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240707134036, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240709124958, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240709124958, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240709132642, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240709132642, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240709183940, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240709183940, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240710155623, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240710155623, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240723102712, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240725152735, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240725152735, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240725182118, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240725182118, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240726100517, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240726100517, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730171504, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730171504, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730174056, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730174056, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730215453, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730215453, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730374423, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240730374423, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240801115359, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240801115359, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240802101043, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240802113716, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240802113716, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240814135330, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240814135330, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240815000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240815000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240815000001, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240816103247, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240816103247, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240820091218, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240826111228, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240826160025, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240826160025, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829165448, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829165448, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829165605, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829165605, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829165715, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829165930, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829165930, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829170023, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829170023, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829170033, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829170033, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240829170044, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240905105135, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240905140514, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240905200000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240905200000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240905200001, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20240905200001, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241002104104, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241002104105, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241002104105, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241002104106, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241002104106, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241002210000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241002210000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241003145349, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241003145349, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241004005000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241004005000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241008083925, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241008083925, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241009090010, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241017163402, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241017163402, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241021224359, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241021224359, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241022140321, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241025111236, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241025112748, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241025141855, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241110152839, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241110152840, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241110152841, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241110152841, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241116233322, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241122171434, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241125150614, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241125150614, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241203125346, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241203125346, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241203130032, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241203130032, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241205122800, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241209164540, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241210140021, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241219180042, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241219180042, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241220100000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241220114903, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241220114903, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241220114904, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241224000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241224000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241230000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20241231112624, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250102121439, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250121094045, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250121094045, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250121094500, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250121094600, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250121094600, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250121094700, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250121094700, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250124194347, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250124194347, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250127162751, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250127162751, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250213104005, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250214205657, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250217093329, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250217093329, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250219090511, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250219100000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250219100000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250219142401, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250224184002, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250225085436, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250226000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250226153445, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250304162702, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250306144233, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250313163430, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250317130944, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250318165922, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250318165922, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250320132525, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250320200000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250320200000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250326161930, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250326161930, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250326161931, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250326161931, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250331042354, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250331154206, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250331154206, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250401155831, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250408133233, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250410104321, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250410104321, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250421085116, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250422095806, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250424153059, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250430103833, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250430112622, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250430112622, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250501162727, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250501162727, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250502154517, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250502222222, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250502222222, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250507170845, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250507170845, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250513162912, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250513162912, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250519161614, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250519161614, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250519170000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250520153848, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250528115932, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250529102706, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250603105558, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250609102714, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250609102714, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250609112613, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250613103810, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250616193950, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250616193950, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250624140757, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250624140757, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250626130239, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250629131032, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250701155654, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250701155654, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250707095725, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250716152435, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250716152435, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250718091828, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250718091828, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250728122229, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250731122715, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250731151000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250731151000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250803000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250805083116, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250805083116, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250807140441, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250808000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250811155036, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250813205039, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250813205039, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250814123333, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250815130115, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250815130115, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250816115553, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250817154557, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250825113751, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250827113140, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250828120836, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250902112642, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250902112642, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250904091745, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250904091745, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250905090000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250905090000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250922083056, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250922083056, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250923120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250926123048, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20250926123048, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103505, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103505, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103600, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103700, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103700, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103800, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103800, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103900, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251015103900, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140100, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140100, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140110, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140110, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140200, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140200, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140300, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140300, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140400, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251028140400, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251031154558, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251031154558, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251103160848, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251103160848, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251104112849, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251106000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251106000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251107164629, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251107164629, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251107170854, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251107170854, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251110172137, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251110172137, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251111153133, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251111153133, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251117020000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251117020100, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251117020100, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251117020200, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251117020200, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251121100000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251121124239, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251121124239, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251124090450, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251124090450, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251124135808, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251124140138, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251124140138, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251124162948, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251124162948, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251127113559, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251202162232, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251203170808, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251203170808, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251207050413, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251207050413, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251208215800, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251209221730, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251209221730, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251209221850, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251215163721, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251217000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251217000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251217120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251217120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251229000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251229000010, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251229000020, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20251229000020, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260106000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260106000000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260108200708, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260108214732, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260109231821, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260109231821, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260113012054, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260113012054, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260124200020, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260124200020, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260126150840, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260126150840, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260126210724, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260202151756, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260205184907, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260210151544, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260210155109, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260210155109, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260210181120, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260210181120, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260211200153, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260211200153, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260217141240, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260217141240, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260217200906, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260218175704, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260314120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120001, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120001, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120002, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120002, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120003, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120004, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120005, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120006, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120006, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120007, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120007, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120008, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120009, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120010, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120010, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260316120011, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260317120000, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260318184559, true); +INSERT INTO migration_status_tables (version_id, is_applied) VALUES (20260318184559, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20161229171615, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20170223171234, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20170301093653, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20170314151620, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20181119180000, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20210330130314, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20210806135609, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20210819120215, true); +INSERT INTO migration_status_data (version_id, is_applied) VALUES (20230525175650, true); +-- Manual fixes for tables the converter can't handle +DROP TABLE IF EXISTS "abm_tokens" CASCADE; +CREATE TABLE "abm_tokens" ( + "id" integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + "organization_name" varchar(255) NOT NULL, + "apple_id" varchar(255) NOT NULL, + "terms_expired" boolean NOT NULL DEFAULT FALSE, + "renew_at" timestamp NOT NULL, + "token" bytea NOT NULL, + "macos_default_team_id" integer DEFAULT NULL, + "ios_default_team_id" integer DEFAULT NULL, + "ipados_default_team_id" integer DEFAULT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "idx_abm_tokens_organization_name" UNIQUE ("organization_name") +); + +DROP TABLE IF EXISTS "carve_metadata" CASCADE; +CREATE TABLE "carve_metadata" ( + "id" integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + "host_id" integer NOT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "name" varchar(255) DEFAULT NULL, + "block_count" integer NOT NULL, + "block_size" integer NOT NULL, + "carve_size" bigint NOT NULL, + "carve_id" varchar(64) NOT NULL, + "request_id" varchar(64) NOT NULL, + "session_id" varchar(255) NOT NULL, + "expired" boolean DEFAULT FALSE, + "max_block" integer DEFAULT -1, + "error" text, + CONSTRAINT "idx_session_id" UNIQUE ("session_id"), + CONSTRAINT "idx_name" UNIQUE ("name") +); + +DROP TABLE IF EXISTS "host_mdm_apple_bootstrap_packages" CASCADE; +CREATE TABLE "host_mdm_apple_bootstrap_packages" ( + "host_uuid" varchar(127) NOT NULL PRIMARY KEY, + "command_uuid" varchar(127) DEFAULT NULL, + "skipped" boolean NOT NULL DEFAULT FALSE, + CONSTRAINT "ck_skipped_or_commanduuid" CHECK (("skipped" = FALSE) = ("command_uuid" IS NOT NULL)) +); + +DROP TABLE IF EXISTS "host_mdm_windows_profiles" CASCADE; +CREATE TABLE "host_mdm_windows_profiles" ( + "host_uuid" varchar(255) NOT NULL, + "status" varchar(20) DEFAULT NULL, + "operation_type" varchar(20) DEFAULT NULL, + "detail" text, + "command_uuid" varchar(127) NOT NULL, + "profile_name" varchar(255) NOT NULL DEFAULT '', + "retries" smallint NOT NULL DEFAULT 0, + "profile_uuid" varchar(37) NOT NULL DEFAULT '', + "checksum" bytea NOT NULL DEFAULT '\x00000000000000000000000000000000', + "secrets_updated_at" timestamp DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("host_uuid","profile_uuid") +); + +DROP TABLE IF EXISTS "mdm_android_configuration_profiles" CASCADE; +CREATE TABLE "mdm_android_configuration_profiles" ( + "profile_uuid" varchar(37) NOT NULL DEFAULT '' PRIMARY KEY, + "team_id" integer NOT NULL DEFAULT 0, + "name" varchar(255) NOT NULL, + "raw_json" jsonb NOT NULL, + "auto_increment" bigint GENERATED BY DEFAULT AS IDENTITY, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "idx_mdm_android_auto_increment" UNIQUE ("auto_increment"), + CONSTRAINT "idx_mdm_android_configuration_profiles_team_id_name" UNIQUE ("team_id","name") +); + +DROP TABLE IF EXISTS "mdm_apple_declarations" CASCADE; +CREATE TABLE "mdm_apple_declarations" ( + "declaration_uuid" varchar(37) NOT NULL DEFAULT '' PRIMARY KEY, + "team_id" integer NOT NULL DEFAULT 0, + "identifier" varchar(255) NOT NULL, + "name" varchar(255) NOT NULL, + "raw_json" text NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp DEFAULT NULL, + "auto_increment" bigint GENERATED BY DEFAULT AS IDENTITY, + "secrets_updated_at" timestamp DEFAULT NULL, + "token" bytea, + "scope" text NOT NULL DEFAULT 'System', + CONSTRAINT "idx_mdm_apple_declaration_team_identifier" UNIQUE ("team_id","identifier"), + CONSTRAINT "idx_mdm_apple_declaration_team_name" UNIQUE ("team_id","name"), + CONSTRAINT "idx_mdm_apple_declarations_auto_increment" UNIQUE ("auto_increment") +); + +DROP TABLE IF EXISTS "mdm_apple_enrollment_profiles" CASCADE; +CREATE TABLE "mdm_apple_enrollment_profiles" ( + "id" integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + "token" varchar(36) DEFAULT NULL, + "type" varchar(10) NOT NULL DEFAULT 'automatic', + "dep_profile" jsonb DEFAULT NULL, + "created_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "idx_enrollment_profiles_type" UNIQUE ("type"), + CONSTRAINT "idx_enrollment_profiles_token" UNIQUE ("token") +); + +DROP TABLE IF EXISTS "mdm_configuration_profile_labels" CASCADE; +CREATE TABLE "mdm_configuration_profile_labels" ( + "id" integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + "apple_profile_uuid" varchar(37) DEFAULT NULL, + "windows_profile_uuid" varchar(37) DEFAULT NULL, + "label_name" varchar(255) NOT NULL, + "label_id" integer DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "exclude" boolean NOT NULL DEFAULT FALSE, + "require_all" boolean NOT NULL DEFAULT FALSE, + "android_profile_uuid" varchar(37) DEFAULT NULL, + CONSTRAINT "idx_mdm_configuration_profile_labels_apple_label_name" UNIQUE ("apple_profile_uuid","label_name"), + CONSTRAINT "idx_mdm_configuration_profile_labels_windows_label_name" UNIQUE ("windows_profile_uuid","label_name"), + CONSTRAINT "idx_mdm_configuration_profile_labels_android_label_name" UNIQUE ("android_profile_uuid","label_name") +); + +DROP TABLE IF EXISTS "mdm_windows_configuration_profiles" CASCADE; +CREATE TABLE "mdm_windows_configuration_profiles" ( + "profile_uuid" varchar(37) NOT NULL DEFAULT '' PRIMARY KEY, + "team_id" integer NOT NULL DEFAULT 0, + "name" varchar(255) NOT NULL, + "syncml" bytea NOT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "uploaded_at" timestamp DEFAULT NULL, + "auto_increment" bigint GENERATED BY DEFAULT AS IDENTITY, + "checksum" bytea, + "secrets_updated_at" timestamp DEFAULT NULL, + CONSTRAINT "idx_mdm_windows_configuration_profiles_team_id_name" UNIQUE ("team_id","name"), + CONSTRAINT "idx_mdm_win_config_auto_increment" UNIQUE ("auto_increment") +); + +DROP TABLE IF EXISTS "operating_system_version_vulnerabilities" CASCADE; +CREATE TABLE "operating_system_version_vulnerabilities" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + "os_version_id" integer NOT NULL, + "cve" varchar(255) NOT NULL, + "team_id" integer DEFAULT NULL, + "source" smallint DEFAULT 0, + "resolved_in_version" varchar(255) DEFAULT NULL, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE UNIQUE INDEX "idx_os_version_vulnerabilities_unq_os_version_team_cve2" ON "operating_system_version_vulnerabilities" ((COALESCE("team_id",-1)),"os_version_id","cve"); + +SELECT 1; + +CREATE TABLE IF NOT EXISTS "host_recovery_key_passwords" ( + "host_uuid" varchar(255) NOT NULL, + "encrypted_password" bytea NOT NULL, + "status" varchar(20) DEFAULT NULL, + "operation_type" varchar(20) NOT NULL, + "error_message" text, + "deleted" boolean NOT NULL DEFAULT FALSE, + "created_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "pending_encrypted_password" bytea, + "pending_error_message" text, + PRIMARY KEY ("host_uuid") +); + +CREATE OR REPLACE VIEW "nano_view_queue" AS +SELECT + q.id AS id, + q.created_at AS created_at, + q.active AS active, + q.priority AS priority, + c.command_uuid AS command_uuid, + c.request_type AS request_type, + c.command AS command, + r.updated_at AS result_updated_at, + r.status AS status, + r.result AS result +FROM + nano_enrollment_queue q + JOIN nano_commands c ON q.command_uuid = c.command_uuid + LEFT JOIN nano_command_results r ON r.command_uuid = q.command_uuid AND r.id = q.id +ORDER BY q.priority DESC, q.created_at; + +/* Trigger: compute declaration token from raw_json and secrets_updated_at */ +CREATE OR REPLACE FUNCTION compute_declaration_token() RETURNS trigger AS $$ +BEGIN + NEW.token := decode(md5(NEW.raw_json || COALESCE(NEW.secrets_updated_at::text, '')), 'hex'); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_compute_declaration_token + BEFORE INSERT OR UPDATE ON "mdm_apple_declarations" + FOR EACH ROW EXECUTE FUNCTION compute_declaration_token(); + +/* Trigger: compute enrollment_status from enrolled, installed_from_dep, is_server, is_personal_enrollment */ +CREATE OR REPLACE FUNCTION compute_enrollment_status() RETURNS trigger AS $$ +BEGIN + NEW.enrollment_status := CASE + WHEN NEW.is_server = TRUE THEN NULL + WHEN NEW.enrolled = TRUE AND NEW.installed_from_dep = FALSE AND NEW.is_personal_enrollment = TRUE THEN 'On (personal)' + WHEN NEW.enrolled = TRUE AND NEW.installed_from_dep = FALSE AND NEW.is_personal_enrollment = FALSE THEN 'On (manual)' + WHEN NEW.enrolled = TRUE AND NEW.installed_from_dep = TRUE AND NEW.is_personal_enrollment = FALSE THEN 'On (automatic)' + WHEN NEW.enrolled = FALSE AND NEW.installed_from_dep = TRUE THEN 'Pending' + WHEN NEW.enrolled = FALSE AND NEW.installed_from_dep = FALSE THEN 'Off' + ELSE NULL + END; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_compute_enrollment_status + BEFORE INSERT OR UPDATE ON "host_mdm" + FOR EACH ROW EXECUTE FUNCTION compute_enrollment_status(); + +/* Trigger: compute execution_status from exit codes, pre_install_query_output, canceled, uninstall */ +CREATE OR REPLACE FUNCTION compute_execution_status() RETURNS trigger AS $$ +BEGIN + NEW.execution_status := CASE + WHEN NEW.canceled = TRUE AND NEW.uninstall = FALSE THEN 'canceled_install' + WHEN NEW.canceled = TRUE AND NEW.uninstall = TRUE THEN 'canceled_uninstall' + WHEN NEW.post_install_script_exit_code IS NOT NULL AND NEW.post_install_script_exit_code = 0 THEN 'installed' + WHEN NEW.post_install_script_exit_code IS NOT NULL AND NEW.post_install_script_exit_code != 0 THEN 'failed_install' + WHEN NEW.install_script_exit_code IS NOT NULL AND NEW.install_script_exit_code = 0 THEN 'installed' + WHEN NEW.install_script_exit_code IS NOT NULL AND NEW.install_script_exit_code != 0 THEN 'failed_install' + WHEN NEW.pre_install_query_output IS NOT NULL AND NEW.pre_install_query_output = '' THEN 'failed_install' + WHEN NEW.host_id IS NOT NULL AND NEW.uninstall = FALSE THEN 'pending_install' + WHEN NEW.uninstall_script_exit_code IS NOT NULL AND NEW.uninstall_script_exit_code != 0 THEN 'failed_uninstall' + WHEN NEW.uninstall_script_exit_code IS NOT NULL AND NEW.uninstall_script_exit_code = 0 THEN NULL + WHEN NEW.host_id IS NOT NULL AND NEW.uninstall = TRUE THEN 'pending_uninstall' + ELSE NULL + END; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_compute_execution_status + BEFORE INSERT OR UPDATE ON "host_software_installs" + FOR EACH ROW EXECUTE FUNCTION compute_execution_status(); + +/* Trigger: compute status (like execution_status but also checks removed flag) */ +CREATE OR REPLACE FUNCTION compute_hsi_status() RETURNS trigger AS $$ +BEGIN + NEW.status := CASE + WHEN NEW.removed = TRUE THEN NULL + WHEN NEW.canceled = TRUE AND NEW.uninstall = FALSE THEN 'canceled_install' + WHEN NEW.canceled = TRUE AND NEW.uninstall = TRUE THEN 'canceled_uninstall' + WHEN NEW.post_install_script_exit_code IS NOT NULL AND NEW.post_install_script_exit_code = 0 THEN 'installed' + WHEN NEW.post_install_script_exit_code IS NOT NULL AND NEW.post_install_script_exit_code != 0 THEN 'failed_install' + WHEN NEW.install_script_exit_code IS NOT NULL AND NEW.install_script_exit_code = 0 THEN 'installed' + WHEN NEW.install_script_exit_code IS NOT NULL AND NEW.install_script_exit_code != 0 THEN 'failed_install' + WHEN NEW.pre_install_query_output IS NOT NULL AND NEW.pre_install_query_output = '' THEN 'failed_install' + WHEN NEW.host_id IS NOT NULL AND NEW.uninstall = FALSE THEN 'pending_install' + WHEN NEW.uninstall_script_exit_code IS NOT NULL AND NEW.uninstall_script_exit_code != 0 THEN 'failed_uninstall' + WHEN NEW.uninstall_script_exit_code IS NOT NULL AND NEW.uninstall_script_exit_code = 0 THEN NULL + WHEN NEW.host_id IS NOT NULL AND NEW.uninstall = TRUE THEN 'pending_uninstall' + ELSE NULL + END; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_compute_hsi_status + BEFORE INSERT OR UPDATE ON "host_software_installs" + FOR EACH ROW EXECUTE FUNCTION compute_hsi_status(); + +/* Seed data: fleet_variables used by tests */ +INSERT INTO "fleet_variables" ("name", "is_prefix", "created_at") VALUES + ('FLEET_VAR_NDES_SCEP_CHALLENGE', FALSE, '2025-04-22 00:00:00'), + ('FLEET_VAR_NDES_SCEP_PROXY_URL', FALSE, '2025-04-22 00:00:00'), + ('FLEET_VAR_HOST_END_USER_EMAIL_IDP', FALSE, '2025-04-22 00:00:00'), + ('FLEET_VAR_HOST_HARDWARE_SERIAL', FALSE, '2025-04-22 00:00:00'), + ('FLEET_VAR_HOST_END_USER_IDP_USERNAME', FALSE, '2025-04-22 00:00:00'), + ('FLEET_VAR_HOST_END_USER_IDP_USERNAME_LOCAL_PART', FALSE, '2025-04-22 00:00:00'), + ('FLEET_VAR_HOST_END_USER_IDP_GROUPS', FALSE, '2025-04-22 00:00:00'), + ('FLEET_VAR_DIGICERT_DATA_', TRUE, '2025-04-22 00:00:00'), + ('FLEET_VAR_DIGICERT_PASSWORD_', TRUE, '2025-04-22 00:00:00'), + ('FLEET_VAR_CUSTOM_SCEP_CHALLENGE_', TRUE, '2025-04-22 00:00:00'), + ('FLEET_VAR_CUSTOM_SCEP_PROXY_URL_', TRUE, '2025-04-22 00:00:00'), + ('FLEET_VAR_SCEP_RENEWAL_ID', FALSE, '2025-04-30 00:00:00'), + ('FLEET_VAR_HOST_END_USER_IDP_DEPARTMENT', FALSE, '2025-06-27 00:00:00'), + ('FLEET_VAR_HOST_UUID', FALSE, '2025-08-08 00:00:00'), + ('FLEET_VAR_HOST_END_USER_IDP_FULL_NAME', FALSE, '2025-08-25 00:00:00'), + ('FLEET_VAR_SCEP_WINDOWS_CERTIFICATE_ID', FALSE, '2025-10-22 00:00:00'), + ('FLEET_VAR_HOST_PLATFORM', FALSE, '2025-11-19 00:00:00'); + +/* Trigger: compute uuid text from uuid_bin bytea for calendar_events */ +CREATE OR REPLACE FUNCTION compute_calendar_event_uuid() RETURNS trigger AS $$ +BEGIN + IF NEW.uuid_bin IS NOT NULL AND length(NEW.uuid_bin) = 16 THEN + NEW.uuid := lower( + encode(substring(NEW.uuid_bin from 1 for 4), 'hex') || '-' || + encode(substring(NEW.uuid_bin from 5 for 2), 'hex') || '-' || + encode(substring(NEW.uuid_bin from 7 for 2), 'hex') || '-' || + encode(substring(NEW.uuid_bin from 9 for 2), 'hex') || '-' || + encode(substring(NEW.uuid_bin from 11 for 6), 'hex') + ); + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_compute_calendar_event_uuid + BEFORE INSERT OR UPDATE ON "calendar_events" + FOR EACH ROW EXECUTE FUNCTION compute_calendar_event_uuid(); diff --git a/server/datastore/mysql/postgres_smoke_test.go b/server/datastore/mysql/postgres_smoke_test.go new file mode 100644 index 00000000000..8a3651f596e --- /dev/null +++ b/server/datastore/mysql/postgres_smoke_test.go @@ -0,0 +1,417 @@ +package mysql + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestPostgresSmokeTest verifies basic PostgreSQL connectivity and dialect +// SQL execution. Requires POSTGRES_TEST=1 and a running postgres_test container. +func TestPostgresSmokeTest(t *testing.T) { + ds := CreatePostgresDS(t) + + // Verify we got a PG-backed datastore + assert.IsType(t, postgresDialect{}, ds.dialect) + + // Create a simple table using PG-native DDL + _, err := ds.primary.Exec(` + CREATE TABLE IF NOT EXISTS pg_smoke_test ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ DEFAULT NOW() + ) + `) + require.NoError(t, err) + + // Insert using the dialect's InsertIgnoreInto (PG: INSERT INTO + ON CONFLICT DO NOTHING) + stmt := ds.dialect.InsertIgnoreInto() + ` pg_smoke_test (name) VALUES ($1)` + ds.dialect.OnConflictDoNothing("name") + _, err = ds.primary.Exec(stmt, "test-host") + require.NoError(t, err) + + // Insert duplicate — should be silently ignored + _, err = ds.primary.Exec(stmt, "test-host") + require.NoError(t, err) + + // Verify only one row + var count int + err = ds.primary.Get(&count, "SELECT COUNT(*) FROM pg_smoke_test WHERE name = $1", "test-host") + require.NoError(t, err) + assert.Equal(t, 1, count) + + // Test upsert via OnDuplicateKey + upsertStmt := `INSERT INTO pg_smoke_test (name) VALUES ($1) ` + + ds.dialect.OnDuplicateKey("name", "name=VALUES(name)") + // Note: For PG this becomes: ON CONFLICT (name) DO UPDATE SET name=EXCLUDED.name + _, err = ds.primary.Exec(upsertStmt, "test-host-2") + require.NoError(t, err) + + // Verify GroupConcat equivalent + _, err = ds.primary.Exec(`INSERT INTO pg_smoke_test (name) VALUES ('a'), ('b'), ('c')`) + require.NoError(t, err) + + var names string + err = ds.primary.Get(&names, "SELECT "+ds.dialect.GroupConcat("name", ",")+" FROM pg_smoke_test") + require.NoError(t, err) + assert.NotEmpty(t, names) + + // Verify JSON operations + _, err = ds.primary.Exec(`CREATE TABLE IF NOT EXISTS pg_json_test (id SERIAL PRIMARY KEY, data JSONB DEFAULT '{}')`) + require.NoError(t, err) + _, err = ds.primary.Exec(`INSERT INTO pg_json_test (data) VALUES ('{"name": "fleet", "version": "4.83"}')`) + require.NoError(t, err) + + var version string + err = ds.primary.Get(&version, "SELECT "+ds.dialect.JSONUnquoteExtract("data", "$.version")+" FROM pg_json_test LIMIT 1") + require.NoError(t, err) + assert.Equal(t, "4.83", version) +} + +func TestPostgresNewHost(t *testing.T) { + ds := CreatePostgresDS(t) + ctx := context.Background() + + host, err := ds.NewHost(ctx, &fleet.Host{ + OsqueryHostID: ptr.String("pg-test-host"), + NodeKey: ptr.String("pg-test-key"), + UUID: "pg-test-uuid", + Hostname: "pg-test-hostname", + Platform: "darwin", + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + }) + if err != nil { + t.Fatalf("NewHost failed: %v", err) + } + assert.NotNil(t, host) + assert.NotZero(t, host.ID) + t.Logf("Created host ID: %d", host.ID) +} + +func TestPostgresNewHostViaTestHelper(t *testing.T) { + ds := CreatePostgresDS(t) + ctx := context.Background() + + // This is how test helpers create hosts - using the test package helper + host := &fleet.Host{ + OsqueryHostID: ptr.String("pg-helper-host"), + NodeKey: ptr.String("pg-helper-key"), + UUID: "pg-helper-uuid", + Hostname: "pg-helper", + Platform: "darwin", + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + } + created, err := ds.NewHost(ctx, host) + require.NoError(t, err, "NewHost should work") + require.NotNil(t, created) + t.Logf("Host created: ID=%d", created.ID) + + // Now try the operations that follow in typical test setup + err = ds.RecordLabelQueryExecutions(ctx, created, map[uint]*bool{}, time.Now(), false) + if err != nil { + t.Logf("RecordLabelQueryExecutions error: %v", err) + } + + // Try saving host users + err = ds.SaveHostUsers(ctx, created.ID, []fleet.HostUser{ + {Username: "testuser", Uid: 1001}, + }) + if err != nil { + t.Logf("SaveHostUsers error: %v", err) + } +} + +// TestPostgresDatastoreOperations exercises a broad set of datastore operations +// against PostgreSQL to find SQL compatibility issues. +func TestPostgresDatastoreOperations(t *testing.T) { + ds := CreatePostgresDS(t) + ctx := context.Background() + + // --- Host CRUD --- + host, err := ds.NewHost(ctx, &fleet.Host{ + OsqueryHostID: ptr.String("pg-ops-host-1"), + NodeKey: ptr.String("pg-ops-key-1"), + UUID: "pg-ops-uuid-1", + Hostname: "pg-ops-hostname-1", + Platform: "darwin", + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + }) + require.NoError(t, err, "NewHost") + + t.Run("HostByIdentifier", func(t *testing.T) { + h, err := ds.HostByIdentifier(ctx, "pg-ops-uuid-1") + if err != nil { + t.Logf("FAIL HostByIdentifier: %v", err) + return + } + assert.Equal(t, host.ID, h.ID) + }) + + t.Run("UpdateHost", func(t *testing.T) { + host.Hostname = "pg-ops-hostname-updated" + err := ds.UpdateHost(ctx, host) + if err != nil { + t.Logf("FAIL UpdateHost: %v", err) + } + }) + + t.Run("Host", func(t *testing.T) { + h, err := ds.Host(ctx, host.ID) + if err != nil { + t.Logf("FAIL Host: %v", err) + return + } + assert.Equal(t, "pg-ops-hostname-updated", h.Hostname) + }) + + // --- Labels --- + t.Run("Labels", func(t *testing.T) { + labels, err := ds.ListLabels(ctx, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String("admin")}}, fleet.ListOptions{}, false) + if err != nil { + t.Logf("FAIL ListLabels: %v", err) + return + } + t.Logf("Labels found: %d", len(labels)) + }) + + t.Run("RecordLabelQueryExecutions", func(t *testing.T) { + trueVal := true + err := ds.RecordLabelQueryExecutions(ctx, host, map[uint]*bool{1: &trueVal}, time.Now(), false) + if err != nil { + t.Logf("FAIL RecordLabelQueryExecutions: %v", err) + } + }) + + // --- Queries --- + t.Run("NewQuery", func(t *testing.T) { + q, err := ds.NewQuery(ctx, &fleet.Query{ + Name: "pg-test-query", + Description: "Test query for PG compat", + Query: "SELECT 1", + Logging: fleet.LoggingSnapshot, + }) + if err != nil { + t.Logf("FAIL NewQuery: %v", err) + return + } + assert.NotZero(t, q.ID) + + // List queries + queries, _, _, _, err := ds.ListQueries(ctx, fleet.ListQueryOptions{ListOptions: fleet.ListOptions{}}) + if err != nil { + t.Logf("FAIL ListQueries: %v", err) + return + } + t.Logf("Queries found: %d", len(queries)) + }) + + // --- Packs --- + t.Run("NewPack", func(t *testing.T) { + p, err := ds.NewPack(ctx, &fleet.Pack{ + Name: "pg-test-pack", + }) + if err != nil { + t.Logf("FAIL NewPack: %v", err) + return + } + assert.NotZero(t, p.ID) + }) + + // --- Users --- + t.Run("NewUser", func(t *testing.T) { + u, err := ds.NewUser(ctx, &fleet.User{ + Name: "pg-test-user", + Email: "pg-test@example.com", + Password: []byte("test-password-hash"), + GlobalRole: ptr.String("admin"), + }) + if err != nil { + t.Logf("FAIL NewUser: %v", err) + return + } + assert.NotZero(t, u.ID) + + // Find user by email + found, err := ds.UserByEmail(ctx, "pg-test@example.com") + if err != nil { + t.Logf("FAIL UserByEmail: %v", err) + return + } + assert.Equal(t, u.ID, found.ID) + }) + + // --- Teams --- + t.Run("NewTeam", func(t *testing.T) { + team, err := ds.NewTeam(ctx, &fleet.Team{ + Name: "pg-test-team", + }) + if err != nil { + t.Logf("FAIL NewTeam: %v", err) + return + } + assert.NotZero(t, team.ID) + }) + + // --- Policies --- + t.Run("NewGlobalPolicy", func(t *testing.T) { + p, err := ds.NewGlobalPolicy(ctx, ptr.Uint(0), fleet.PolicyPayload{ + Name: "pg-test-policy", + Query: "SELECT 1", + }) + if err != nil { + t.Logf("FAIL NewGlobalPolicy: %v", err) + return + } + assert.NotZero(t, p.ID) + }) + + // --- Host additional data --- + t.Run("SaveHostAdditional", func(t *testing.T) { + additional := json.RawMessage(`{"test_field": "test_value"}`) + err := ds.SaveHostAdditional(ctx, host.ID, &additional) + if err != nil { + t.Logf("FAIL SaveHostAdditional: %v", err) + } + }) + + // --- Software --- + t.Run("UpdateHostSoftware", func(t *testing.T) { + sw := []fleet.Software{ + {Name: "pg-test-sw", Version: "1.0", Source: "test"}, + } + _, err := ds.UpdateHostSoftware(ctx, host.ID, sw) + if err != nil { + t.Logf("FAIL UpdateHostSoftware: %v", err) + } + }) + + // --- Sessions --- + t.Run("NewSession", func(t *testing.T) { + users, err := ds.ListUsers(ctx, fleet.UserListOptions{ListOptions: fleet.ListOptions{}}) + if err != nil || len(users) == 0 { + t.Logf("SKIP NewSession: no users") + return + } + sess, err := ds.NewSession(ctx, users[0].ID, 64) + if err != nil { + t.Logf("FAIL NewSession: %v", err) + return + } + assert.NotZero(t, sess.ID) + }) + + // --- Enroll secrets --- + t.Run("ApplyEnrollSecrets", func(t *testing.T) { + err := ds.ApplyEnrollSecrets(ctx, nil, []*fleet.EnrollSecret{ + {Secret: "pg-test-secret"}, + }) + if err != nil { + t.Logf("FAIL ApplyEnrollSecrets: %v", err) + } + }) + + // --- App config --- + t.Run("AppConfig", func(t *testing.T) { + cfg, err := ds.AppConfig(ctx) + if err != nil { + t.Logf("FAIL AppConfig: %v", err) + return + } + assert.NotNil(t, cfg) + }) + + // --- ListHosts --- + t.Run("ListHosts", func(t *testing.T) { + hosts, err := ds.ListHosts(ctx, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String("admin")}}, fleet.HostListOptions{ListOptions: fleet.ListOptions{}}) + if err != nil { + t.Logf("FAIL ListHosts: %v", err) + return + } + assert.GreaterOrEqual(t, len(hosts), 1) + }) + + // --- CountHosts --- + t.Run("CountHosts", func(t *testing.T) { + count, err := ds.CountHosts(ctx, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String("admin")}}, fleet.HostListOptions{}) + if err != nil { + t.Logf("FAIL CountHosts: %v", err) + return + } + assert.GreaterOrEqual(t, count, 1) + }) + + t.Run("HostLite", func(t *testing.T) { + h, err := ds.HostLite(ctx, host.ID) + if err != nil { + t.Logf("FAIL HostLite: %v", err) + return + } + assert.Equal(t, host.ID, h.ID) + }) + + // --- Targets --- + t.Run("CountHostsInTargets", func(t *testing.T) { + metrics, err := ds.CountHostsInTargets(ctx, + fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String("admin")}}, + fleet.HostTargets{HostIDs: []uint{host.ID}}, + time.Now(), + ) + if err != nil { + t.Logf("FAIL CountHostsInTargets: %v", err) + return + } + assert.GreaterOrEqual(t, metrics.TotalHosts, uint(1)) + }) + + // --- Host disk encryption key --- + t.Run("SetOrUpdateHostDiskEncryptionKey", func(t *testing.T) { + _, err := ds.SetOrUpdateHostDiskEncryptionKey(ctx, host, "test-key", "test-client", ptr.Bool(false)) + if err != nil { + t.Logf("FAIL SetOrUpdateHostDiskEncryptionKey: %v", err) + } + }) + + // --- Cron stats --- + t.Run("InsertCronStats", func(t *testing.T) { + id, err := ds.InsertCronStats(ctx, fleet.CronStatsTypeScheduled, "test-cron", "test-instance", fleet.CronStatsStatusPending) + if err != nil { + t.Logf("FAIL InsertCronStats: %v", err) + return + } + assert.NotZero(t, id) + }) + + // --- ListPolicies --- + t.Run("ListGlobalPolicies", func(t *testing.T) { + policies, err := ds.ListGlobalPolicies(ctx, fleet.ListOptions{}) + if err != nil { + t.Logf("FAIL ListGlobalPolicies: %v", err) + return + } + assert.GreaterOrEqual(t, len(policies), 1) + }) + + // --- Invites --- + t.Run("ListInvites", func(t *testing.T) { + invites, err := ds.ListInvites(ctx, fleet.ListOptions{}) + if err != nil { + t.Logf("FAIL ListInvites: %v", err) + return + } + _ = invites + }) +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 86e6144f264..ad4c9d13957 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -987,12 +987,10 @@ CREATE TABLE `host_recovery_key_passwords` ( `updated_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), `pending_encrypted_password` blob, `pending_error_message` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci, - `auto_rotate_at` timestamp(6) NULL DEFAULT NULL, PRIMARY KEY (`host_uuid`), KEY `status` (`status`), KEY `operation_type` (`operation_type`), KEY `deleted` (`deleted`), - KEY `idx_auto_rotate_at` (`auto_rotate_at`), CONSTRAINT `host_recovery_key_passwords_ibfk_1` FOREIGN KEY (`status`) REFERENCES `mdm_delivery_status` (`status`) ON UPDATE CASCADE, CONSTRAINT `host_recovery_key_passwords_ibfk_2` FOREIGN KEY (`operation_type`) REFERENCES `mdm_operation_types` (`operation_type`) ON UPDATE CASCADE ) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; @@ -1807,9 +1805,9 @@ CREATE TABLE `migration_status_tables` ( `is_applied` tinyint(1) NOT NULL, `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=504 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=501 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'),(350,20250124194347,1,'2020-01-01 01:01:01'),(351,20250127162751,1,'2020-01-01 01:01:01'),(352,20250213104005,1,'2020-01-01 01:01:01'),(353,20250214205657,1,'2020-01-01 01:01:01'),(354,20250217093329,1,'2020-01-01 01:01:01'),(355,20250219090511,1,'2020-01-01 01:01:01'),(356,20250219100000,1,'2020-01-01 01:01:01'),(357,20250219142401,1,'2020-01-01 01:01:01'),(358,20250224184002,1,'2020-01-01 01:01:01'),(359,20250225085436,1,'2020-01-01 01:01:01'),(360,20250226000000,1,'2020-01-01 01:01:01'),(361,20250226153445,1,'2020-01-01 01:01:01'),(362,20250304162702,1,'2020-01-01 01:01:01'),(363,20250306144233,1,'2020-01-01 01:01:01'),(364,20250313163430,1,'2020-01-01 01:01:01'),(365,20250317130944,1,'2020-01-01 01:01:01'),(366,20250318165922,1,'2020-01-01 01:01:01'),(367,20250320132525,1,'2020-01-01 01:01:01'),(368,20250320200000,1,'2020-01-01 01:01:01'),(369,20250326161930,1,'2020-01-01 01:01:01'),(370,20250326161931,1,'2020-01-01 01:01:01'),(371,20250331042354,1,'2020-01-01 01:01:01'),(372,20250331154206,1,'2020-01-01 01:01:01'),(373,20250401155831,1,'2020-01-01 01:01:01'),(374,20250408133233,1,'2020-01-01 01:01:01'),(375,20250410104321,1,'2020-01-01 01:01:01'),(376,20250421085116,1,'2020-01-01 01:01:01'),(377,20250422095806,1,'2020-01-01 01:01:01'),(378,20250424153059,1,'2020-01-01 01:01:01'),(379,20250430103833,1,'2020-01-01 01:01:01'),(380,20250430112622,1,'2020-01-01 01:01:01'),(381,20250501162727,1,'2020-01-01 01:01:01'),(382,20250502154517,1,'2020-01-01 01:01:01'),(383,20250502222222,1,'2020-01-01 01:01:01'),(384,20250507170845,1,'2020-01-01 01:01:01'),(385,20250513162912,1,'2020-01-01 01:01:01'),(386,20250519161614,1,'2020-01-01 01:01:01'),(387,20250519170000,1,'2020-01-01 01:01:01'),(388,20250520153848,1,'2020-01-01 01:01:01'),(389,20250528115932,1,'2020-01-01 01:01:01'),(390,20250529102706,1,'2020-01-01 01:01:01'),(391,20250603105558,1,'2020-01-01 01:01:01'),(392,20250609102714,1,'2020-01-01 01:01:01'),(393,20250609112613,1,'2020-01-01 01:01:01'),(394,20250613103810,1,'2020-01-01 01:01:01'),(395,20250616193950,1,'2020-01-01 01:01:01'),(396,20250624140757,1,'2020-01-01 01:01:01'),(397,20250626130239,1,'2020-01-01 01:01:01'),(398,20250629131032,1,'2020-01-01 01:01:01'),(399,20250701155654,1,'2020-01-01 01:01:01'),(400,20250707095725,1,'2020-01-01 01:01:01'),(401,20250716152435,1,'2020-01-01 01:01:01'),(402,20250718091828,1,'2020-01-01 01:01:01'),(403,20250728122229,1,'2020-01-01 01:01:01'),(404,20250731122715,1,'2020-01-01 01:01:01'),(405,20250731151000,1,'2020-01-01 01:01:01'),(406,20250803000000,1,'2020-01-01 01:01:01'),(407,20250805083116,1,'2020-01-01 01:01:01'),(408,20250807140441,1,'2020-01-01 01:01:01'),(409,20250808000000,1,'2020-01-01 01:01:01'),(410,20250811155036,1,'2020-01-01 01:01:01'),(411,20250813205039,1,'2020-01-01 01:01:01'),(412,20250814123333,1,'2020-01-01 01:01:01'),(413,20250815130115,1,'2020-01-01 01:01:01'),(414,20250816115553,1,'2020-01-01 01:01:01'),(415,20250817154557,1,'2020-01-01 01:01:01'),(416,20250825113751,1,'2020-01-01 01:01:01'),(417,20250827113140,1,'2020-01-01 01:01:01'),(418,20250828120836,1,'2020-01-01 01:01:01'),(419,20250902112642,1,'2020-01-01 01:01:01'),(420,20250904091745,1,'2020-01-01 01:01:01'),(421,20250905090000,1,'2020-01-01 01:01:01'),(422,20250922083056,1,'2020-01-01 01:01:01'),(423,20250923120000,1,'2020-01-01 01:01:01'),(424,20250926123048,1,'2020-01-01 01:01:01'),(425,20251015103505,1,'2020-01-01 01:01:01'),(426,20251015103600,1,'2020-01-01 01:01:01'),(427,20251015103700,1,'2020-01-01 01:01:01'),(428,20251015103800,1,'2020-01-01 01:01:01'),(429,20251015103900,1,'2020-01-01 01:01:01'),(430,20251028140000,1,'2020-01-01 01:01:01'),(431,20251028140100,1,'2020-01-01 01:01:01'),(432,20251028140110,1,'2020-01-01 01:01:01'),(433,20251028140200,1,'2020-01-01 01:01:01'),(434,20251028140300,1,'2020-01-01 01:01:01'),(435,20251028140400,1,'2020-01-01 01:01:01'),(436,20251031154558,1,'2020-01-01 01:01:01'),(437,20251103160848,1,'2020-01-01 01:01:01'),(438,20251104112849,1,'2020-01-01 01:01:01'),(439,20251106000000,1,'2020-01-01 01:01:01'),(440,20251107164629,1,'2020-01-01 01:01:01'),(441,20251107170854,1,'2020-01-01 01:01:01'),(442,20251110172137,1,'2020-01-01 01:01:01'),(443,20251111153133,1,'2020-01-01 01:01:01'),(444,20251117020000,1,'2020-01-01 01:01:01'),(445,20251117020100,1,'2020-01-01 01:01:01'),(446,20251117020200,1,'2020-01-01 01:01:01'),(447,20251121100000,1,'2020-01-01 01:01:01'),(448,20251121124239,1,'2020-01-01 01:01:01'),(449,20251124090450,1,'2020-01-01 01:01:01'),(450,20251124135808,1,'2020-01-01 01:01:01'),(451,20251124140138,1,'2020-01-01 01:01:01'),(452,20251124162948,1,'2020-01-01 01:01:01'),(453,20251127113559,1,'2020-01-01 01:01:01'),(454,20251202162232,1,'2020-01-01 01:01:01'),(455,20251203170808,1,'2020-01-01 01:01:01'),(456,20251207050413,1,'2020-01-01 01:01:01'),(457,20251208215800,1,'2020-01-01 01:01:01'),(458,20251209221730,1,'2020-01-01 01:01:01'),(459,20251209221850,1,'2020-01-01 01:01:01'),(460,20251215163721,1,'2020-01-01 01:01:01'),(461,20251217000000,1,'2020-01-01 01:01:01'),(462,20251217120000,1,'2020-01-01 01:01:01'),(463,20251229000000,1,'2020-01-01 01:01:01'),(464,20251229000010,1,'2020-01-01 01:01:01'),(465,20251229000020,1,'2020-01-01 01:01:01'),(466,20260106000000,1,'2020-01-01 01:01:01'),(467,20260108200708,1,'2020-01-01 01:01:01'),(468,20260108214732,1,'2020-01-01 01:01:01'),(469,20260109231821,1,'2020-01-01 01:01:01'),(470,20260113012054,1,'2020-01-01 01:01:01'),(471,20260124200020,1,'2020-01-01 01:01:01'),(472,20260126150840,1,'2020-01-01 01:01:01'),(473,20260126210724,1,'2020-01-01 01:01:01'),(474,20260202151756,1,'2020-01-01 01:01:01'),(475,20260205184907,1,'2020-01-01 01:01:01'),(476,20260210151544,1,'2020-01-01 01:01:01'),(477,20260210155109,1,'2020-01-01 01:01:01'),(478,20260210181120,1,'2020-01-01 01:01:01'),(479,20260211200153,1,'2020-01-01 01:01:01'),(480,20260217141240,1,'2020-01-01 01:01:01'),(481,20260217200906,1,'2020-01-01 01:01:01'),(482,20260218175704,1,'2020-01-01 01:01:01'),(483,20260314120000,1,'2020-01-01 01:01:01'),(484,20260316120000,1,'2020-01-01 01:01:01'),(485,20260316120001,1,'2020-01-01 01:01:01'),(486,20260316120002,1,'2020-01-01 01:01:01'),(487,20260316120003,1,'2020-01-01 01:01:01'),(488,20260316120004,1,'2020-01-01 01:01:01'),(489,20260316120005,1,'2020-01-01 01:01:01'),(490,20260316120006,1,'2020-01-01 01:01:01'),(491,20260316120007,1,'2020-01-01 01:01:01'),(492,20260316120008,1,'2020-01-01 01:01:01'),(493,20260316120009,1,'2020-01-01 01:01:01'),(494,20260316120010,1,'2020-01-01 01:01:01'),(495,20260316120011,1,'2020-01-01 01:01:01'),(496,20260317120000,1,'2020-01-01 01:01:01'),(497,20260318184559,1,'2020-01-01 01:01:01'),(498,20260319120000,1,'2020-01-01 01:01:01'),(499,20260323144117,1,'2020-01-01 01:01:01'),(500,20260324161944,1,'2020-01-01 01:01:01'),(501,20260324223334,1,'2020-01-01 01:01:01'),(502,20260326131501,1,'2020-01-01 01:01:01'),(503,20260326210603,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'),(350,20250124194347,1,'2020-01-01 01:01:01'),(351,20250127162751,1,'2020-01-01 01:01:01'),(352,20250213104005,1,'2020-01-01 01:01:01'),(353,20250214205657,1,'2020-01-01 01:01:01'),(354,20250217093329,1,'2020-01-01 01:01:01'),(355,20250219090511,1,'2020-01-01 01:01:01'),(356,20250219100000,1,'2020-01-01 01:01:01'),(357,20250219142401,1,'2020-01-01 01:01:01'),(358,20250224184002,1,'2020-01-01 01:01:01'),(359,20250225085436,1,'2020-01-01 01:01:01'),(360,20250226000000,1,'2020-01-01 01:01:01'),(361,20250226153445,1,'2020-01-01 01:01:01'),(362,20250304162702,1,'2020-01-01 01:01:01'),(363,20250306144233,1,'2020-01-01 01:01:01'),(364,20250313163430,1,'2020-01-01 01:01:01'),(365,20250317130944,1,'2020-01-01 01:01:01'),(366,20250318165922,1,'2020-01-01 01:01:01'),(367,20250320132525,1,'2020-01-01 01:01:01'),(368,20250320200000,1,'2020-01-01 01:01:01'),(369,20250326161930,1,'2020-01-01 01:01:01'),(370,20250326161931,1,'2020-01-01 01:01:01'),(371,20250331042354,1,'2020-01-01 01:01:01'),(372,20250331154206,1,'2020-01-01 01:01:01'),(373,20250401155831,1,'2020-01-01 01:01:01'),(374,20250408133233,1,'2020-01-01 01:01:01'),(375,20250410104321,1,'2020-01-01 01:01:01'),(376,20250421085116,1,'2020-01-01 01:01:01'),(377,20250422095806,1,'2020-01-01 01:01:01'),(378,20250424153059,1,'2020-01-01 01:01:01'),(379,20250430103833,1,'2020-01-01 01:01:01'),(380,20250430112622,1,'2020-01-01 01:01:01'),(381,20250501162727,1,'2020-01-01 01:01:01'),(382,20250502154517,1,'2020-01-01 01:01:01'),(383,20250502222222,1,'2020-01-01 01:01:01'),(384,20250507170845,1,'2020-01-01 01:01:01'),(385,20250513162912,1,'2020-01-01 01:01:01'),(386,20250519161614,1,'2020-01-01 01:01:01'),(387,20250519170000,1,'2020-01-01 01:01:01'),(388,20250520153848,1,'2020-01-01 01:01:01'),(389,20250528115932,1,'2020-01-01 01:01:01'),(390,20250529102706,1,'2020-01-01 01:01:01'),(391,20250603105558,1,'2020-01-01 01:01:01'),(392,20250609102714,1,'2020-01-01 01:01:01'),(393,20250609112613,1,'2020-01-01 01:01:01'),(394,20250613103810,1,'2020-01-01 01:01:01'),(395,20250616193950,1,'2020-01-01 01:01:01'),(396,20250624140757,1,'2020-01-01 01:01:01'),(397,20250626130239,1,'2020-01-01 01:01:01'),(398,20250629131032,1,'2020-01-01 01:01:01'),(399,20250701155654,1,'2020-01-01 01:01:01'),(400,20250707095725,1,'2020-01-01 01:01:01'),(401,20250716152435,1,'2020-01-01 01:01:01'),(402,20250718091828,1,'2020-01-01 01:01:01'),(403,20250728122229,1,'2020-01-01 01:01:01'),(404,20250731122715,1,'2020-01-01 01:01:01'),(405,20250731151000,1,'2020-01-01 01:01:01'),(406,20250803000000,1,'2020-01-01 01:01:01'),(407,20250805083116,1,'2020-01-01 01:01:01'),(408,20250807140441,1,'2020-01-01 01:01:01'),(409,20250808000000,1,'2020-01-01 01:01:01'),(410,20250811155036,1,'2020-01-01 01:01:01'),(411,20250813205039,1,'2020-01-01 01:01:01'),(412,20250814123333,1,'2020-01-01 01:01:01'),(413,20250815130115,1,'2020-01-01 01:01:01'),(414,20250816115553,1,'2020-01-01 01:01:01'),(415,20250817154557,1,'2020-01-01 01:01:01'),(416,20250825113751,1,'2020-01-01 01:01:01'),(417,20250827113140,1,'2020-01-01 01:01:01'),(418,20250828120836,1,'2020-01-01 01:01:01'),(419,20250902112642,1,'2020-01-01 01:01:01'),(420,20250904091745,1,'2020-01-01 01:01:01'),(421,20250905090000,1,'2020-01-01 01:01:01'),(422,20250922083056,1,'2020-01-01 01:01:01'),(423,20250923120000,1,'2020-01-01 01:01:01'),(424,20250926123048,1,'2020-01-01 01:01:01'),(425,20251015103505,1,'2020-01-01 01:01:01'),(426,20251015103600,1,'2020-01-01 01:01:01'),(427,20251015103700,1,'2020-01-01 01:01:01'),(428,20251015103800,1,'2020-01-01 01:01:01'),(429,20251015103900,1,'2020-01-01 01:01:01'),(430,20251028140000,1,'2020-01-01 01:01:01'),(431,20251028140100,1,'2020-01-01 01:01:01'),(432,20251028140110,1,'2020-01-01 01:01:01'),(433,20251028140200,1,'2020-01-01 01:01:01'),(434,20251028140300,1,'2020-01-01 01:01:01'),(435,20251028140400,1,'2020-01-01 01:01:01'),(436,20251031154558,1,'2020-01-01 01:01:01'),(437,20251103160848,1,'2020-01-01 01:01:01'),(438,20251104112849,1,'2020-01-01 01:01:01'),(439,20251106000000,1,'2020-01-01 01:01:01'),(440,20251107164629,1,'2020-01-01 01:01:01'),(441,20251107170854,1,'2020-01-01 01:01:01'),(442,20251110172137,1,'2020-01-01 01:01:01'),(443,20251111153133,1,'2020-01-01 01:01:01'),(444,20251117020000,1,'2020-01-01 01:01:01'),(445,20251117020100,1,'2020-01-01 01:01:01'),(446,20251117020200,1,'2020-01-01 01:01:01'),(447,20251121100000,1,'2020-01-01 01:01:01'),(448,20251121124239,1,'2020-01-01 01:01:01'),(449,20251124090450,1,'2020-01-01 01:01:01'),(450,20251124135808,1,'2020-01-01 01:01:01'),(451,20251124140138,1,'2020-01-01 01:01:01'),(452,20251124162948,1,'2020-01-01 01:01:01'),(453,20251127113559,1,'2020-01-01 01:01:01'),(454,20251202162232,1,'2020-01-01 01:01:01'),(455,20251203170808,1,'2020-01-01 01:01:01'),(456,20251207050413,1,'2020-01-01 01:01:01'),(457,20251208215800,1,'2020-01-01 01:01:01'),(458,20251209221730,1,'2020-01-01 01:01:01'),(459,20251209221850,1,'2020-01-01 01:01:01'),(460,20251215163721,1,'2020-01-01 01:01:01'),(461,20251217000000,1,'2020-01-01 01:01:01'),(462,20251217120000,1,'2020-01-01 01:01:01'),(463,20251229000000,1,'2020-01-01 01:01:01'),(464,20251229000010,1,'2020-01-01 01:01:01'),(465,20251229000020,1,'2020-01-01 01:01:01'),(466,20260106000000,1,'2020-01-01 01:01:01'),(467,20260108200708,1,'2020-01-01 01:01:01'),(468,20260108214732,1,'2020-01-01 01:01:01'),(469,20260109231821,1,'2020-01-01 01:01:01'),(470,20260113012054,1,'2020-01-01 01:01:01'),(471,20260124200020,1,'2020-01-01 01:01:01'),(472,20260126150840,1,'2020-01-01 01:01:01'),(473,20260126210724,1,'2020-01-01 01:01:01'),(474,20260202151756,1,'2020-01-01 01:01:01'),(475,20260205184907,1,'2020-01-01 01:01:01'),(476,20260210151544,1,'2020-01-01 01:01:01'),(477,20260210155109,1,'2020-01-01 01:01:01'),(478,20260210181120,1,'2020-01-01 01:01:01'),(479,20260211200153,1,'2020-01-01 01:01:01'),(480,20260217141240,1,'2020-01-01 01:01:01'),(481,20260217200906,1,'2020-01-01 01:01:01'),(482,20260218175704,1,'2020-01-01 01:01:01'),(483,20260314120000,1,'2020-01-01 01:01:01'),(484,20260316120000,1,'2020-01-01 01:01:01'),(485,20260316120001,1,'2020-01-01 01:01:01'),(486,20260316120002,1,'2020-01-01 01:01:01'),(487,20260316120003,1,'2020-01-01 01:01:01'),(488,20260316120004,1,'2020-01-01 01:01:01'),(489,20260316120005,1,'2020-01-01 01:01:01'),(490,20260316120006,1,'2020-01-01 01:01:01'),(491,20260316120007,1,'2020-01-01 01:01:01'),(492,20260316120008,1,'2020-01-01 01:01:01'),(493,20260316120009,1,'2020-01-01 01:01:01'),(494,20260316120010,1,'2020-01-01 01:01:01'),(495,20260316120011,1,'2020-01-01 01:01:01'),(496,20260317120000,1,'2020-01-01 01:01:01'),(497,20260318184559,1,'2020-01-01 01:01:01'),(498,20260319120000,1,'2020-01-01 01:01:01'),(499,20260323144117,1,'2020-01-01 01:01:01'),(500,20260324161944,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `mobile_device_management_solutions` ( @@ -2317,10 +2315,8 @@ CREATE TABLE `query_results` ( `error` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci, `last_fetched` timestamp NOT NULL, `data` json DEFAULT NULL, - `has_data` tinyint(1) GENERATED ALWAYS AS ((`data` is not null)) VIRTUAL, PRIMARY KEY (`id`), - KEY `idx_query_id_host_id_last_fetched` (`query_id`,`host_id`,`last_fetched`), - KEY `idx_query_id_has_data_host_id_last_fetched` (`query_id`,`has_data`,`host_id`,`last_fetched`) + KEY `idx_query_id_host_id_last_fetched` (`query_id`,`host_id`,`last_fetched`) ) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; diff --git a/server/datastore/mysql/testing_utils.go b/server/datastore/mysql/testing_utils.go index d2c996fa944..6cd3c594c50 100644 --- a/server/datastore/mysql/testing_utils.go +++ b/server/datastore/mysql/testing_utils.go @@ -17,7 +17,9 @@ import ( "log/slog" "os" "os/exec" + "path/filepath" "regexp" + "runtime" "strconv" "strings" "sync" @@ -39,6 +41,7 @@ import ( common_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql" "github.com/fleetdm/fleet/v4/server/platform/mysql/testing_utils" "github.com/google/uuid" + _ "github.com/jackc/pgx/v5/stdlib" // register pgx driver for PostgreSQL tests "github.com/jmoiron/sqlx" "github.com/olekukonko/tablewriter" "github.com/smallstep/pkcs7" @@ -413,6 +416,22 @@ func CreateMySQLDS(t testing.TB) *Datastore { return createMySQLDSWithOptions(t, nil) } +// CreateDS creates a test Datastore for the active test database backend. +// When MYSQL_TEST=1 is set, returns a MySQL-backed datastore. +// When POSTGRES_TEST=1 is set, returns a PostgreSQL-backed datastore. +// Skips the test if neither is set. +func CreateDS(t *testing.T) *Datastore { + _, hasMysql := os.LookupEnv("MYSQL_TEST") + _, hasPG := os.LookupEnv("POSTGRES_TEST") + if !hasMysql && !hasPG { + t.Skip("Neither MYSQL_TEST nor POSTGRES_TEST is set") + } + if hasPG { + return CreatePostgresDS(t) + } + return createMySQLDSWithOptions(t, nil) +} + func CreateNamedMySQLDS(t *testing.T, name string) *Datastore { ds, _ := CreateNamedMySQLDSWithConns(t, name) return ds @@ -432,6 +451,195 @@ func CreateNamedMySQLDSWithConns(t *testing.T, name string) (*Datastore, *common return ds, TestDBConnections(t, ds) } +// pgBaselineSchema is loaded once from pg_baseline_schema.sql +var pgBaselineSchema string +var pgBaselineOnce sync.Once + +func loadPGBaselineSchema() string { + pgBaselineOnce.Do(func() { + // Try multiple paths since tests run from different working directories + paths := []string{ + "pg_baseline_schema.sql", + "server/datastore/mysql/pg_baseline_schema.sql", + "../../../server/datastore/mysql/pg_baseline_schema.sql", + } + // Also try relative to the test binary via runtime.Caller + _, thisFile, _, _ := runtime.Caller(0) + if thisFile != "" { + dir := filepath.Dir(thisFile) + paths = append(paths, filepath.Join(dir, "pg_baseline_schema.sql")) + } + for _, p := range paths { + data, err := os.ReadFile(p) + if err == nil { + pgBaselineSchema = string(data) + return + } + } + panic("cannot load pg_baseline_schema.sql from any known path") + }) + return pgBaselineSchema +} + +// CreatePostgresDS creates a test Datastore backed by PostgreSQL. +// Requires POSTGRES_TEST=1 and a running postgres_test container (default port 5434). +// The database is created fresh for each test with the full Fleet schema applied. +func CreatePostgresDS(t *testing.T) *Datastore { + if _, ok := os.LookupEnv("POSTGRES_TEST"); !ok { + t.Skip("PostgreSQL tests are disabled") + } + + port := os.Getenv("FLEET_POSTGRES_TEST_PORT") + if port == "" { + port = "5434" + } + + // Sanitize test name into a valid PG identifier (alphanumeric + underscore only). + dbName := strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '_' { + return r + } + if r >= 'A' && r <= 'Z' { + return r + ('a' - 'A') // lowercase + } + return '_' + }, t.Name()) + if len(dbName) > 63 { + dbName = dbName[:63] // PG identifier limit + } + + // Connect to default db to create test database + adminDSN := fmt.Sprintf("host=localhost port=%s user=fleet password=insecure dbname=fleet sslmode=disable", port) + adminDB, err := sqlx.Open("pgx-rebind", adminDSN) + require.NoError(t, err) + defer adminDB.Close() + + _, _ = adminDB.Exec("DROP DATABASE IF EXISTS " + dbName) + _, err = adminDB.Exec("CREATE DATABASE " + dbName) + require.NoError(t, err) + // Set the test database timezone to UTC so that timestamp columns + // round-trip correctly (PG timestamp without time zone uses session tz). + _, err = adminDB.Exec("ALTER DATABASE " + dbName + " SET timezone TO 'UTC'") + require.NoError(t, err) + + t.Cleanup(func() { + _, _ = adminDB.Exec("DROP DATABASE IF EXISTS " + dbName) + }) + + // Connect to the test database + testDSN := fmt.Sprintf("host=localhost port=%s user=fleet password=insecure dbname=%s sslmode=disable", port, dbName) + testDB, err := sqlx.Open("pgx-rebind", testDSN) + require.NoError(t, err) + + // Apply the baseline schema statement-by-statement. + // Split on ";\n" for statement boundaries, but NOT inside $$ dollar-quoted blocks + // (used by PL/pgSQL trigger functions). + schema := loadPGBaselineSchema() + var stmts []string + inDollarQuote := false + var current strings.Builder + for _, line := range strings.Split(schema, "\n") { + trimmed := strings.TrimSpace(line) + // Count $$ occurrences — odd count toggles dollar-quote state + if strings.Count(trimmed, "$$")%2 == 1 { + inDollarQuote = !inDollarQuote + } + current.WriteString(line) + current.WriteString("\n") + if !inDollarQuote && strings.HasSuffix(trimmed, ";") { + stmts = append(stmts, current.String()) + current.Reset() + } + } + if s := strings.TrimSpace(current.String()); s != "" { + stmts = append(stmts, s) + } + errCount := 0 + for _, stmt := range stmts { + stmt = strings.TrimSpace(stmt) + if stmt == "" { + continue + } + // Strip leading comment lines + for strings.HasPrefix(stmt, "--") { + nl := strings.Index(stmt, "\n") + if nl < 0 { + stmt = "" + break + } + stmt = strings.TrimSpace(stmt[nl+1:]) + } + if stmt == "" { + continue + } + execStmt := stmt + if !strings.HasSuffix(strings.TrimSpace(stmt), ";") { + execStmt = stmt + ";" + } + if _, err := testDB.DB.Exec(execStmt); err != nil { + errCount++ + if errCount <= 3 { + first := stmt + if len(first) > 150 { + first = first[:150] + } + t.Logf("PG schema warning (%d): %v [%s]", errCount, err, first) + } + } + } + if errCount > 0 { + t.Logf("PG schema: %d/%d stmts had errors (non-fatal)", errCount, len(stmts)) + } + + // Verify minimum table count + var tableCount int + if err := testDB.Get(&tableCount, "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='public'"); err == nil { + if tableCount < 180 { + t.Fatalf("PG schema incomplete: only %d tables (expected 190+)", tableCount) + } + } + + // Insert required seed data (app_config_json needs at least one row) + _, _ = testDB.Exec(`INSERT INTO app_config_json (id, json_value) VALUES (1, '{}') ON CONFLICT (id) DO NOTHING`) + // Insert built-in labels that migrations would normally create + if _, err := testDB.Exec(`INSERT INTO labels (name, query, label_type, label_membership_type) VALUES + ('All Hosts', 'SELECT 1', 1, 0), + ('macOS', 'SELECT 1', 1, 0), + ('Ubuntu Linux', 'SELECT 1', 1, 0), + ('CentOS Linux', 'SELECT 1', 1, 0), + ('Windows', 'SELECT 1', 1, 0), + ('Red Hat Linux', 'SELECT 1', 1, 0), + ('All Linux', 'SELECT 1', 1, 0), + ('chrome', 'SELECT 1', 1, 0), + ('iOS', 'SELECT 1', 1, 0), + ('iPadOS', 'SELECT 1', 1, 0), + ('Fedora Linux', 'SELECT 1', 1, 0) + ON CONFLICT (name) DO NOTHING`); err != nil { + t.Logf("PG seed data: labels insert error: %v", err) + } + // Insert mdm delivery status and operation type seed data + _, _ = testDB.Exec(`INSERT INTO mdm_delivery_status (status) VALUES ('failed'), ('applied'), ('pending'), ('verified'), ('verifying') ON CONFLICT (status) DO NOTHING`) + _, _ = testDB.Exec(`INSERT INTO mdm_operation_types (operation_type) VALUES ('install'), ('remove') ON CONFLICT (operation_type) DO NOTHING`) + + logger := slog.New(slog.DiscardHandler) + ds := &Datastore{ + primary: testDB, + replica: testDB, + logger: logger, + clock: clock.C, + dialect: postgresDialect{}, + writeCh: make(chan itemToWrite), + serverPrivateKey: "test-private-key-for-pg-tests!!!", // 32 bytes for AES-256 + stmtCache: make(map[string]*sqlx.Stmt), + } + ds.Datastore = NewAndroidDatastore(logger, testDB, testDB, postgresDialect{}) + t.Cleanup(func() { ds.Close() }) + + go ds.writeChanLoop() + + return ds +} + func ExecAdhocSQL(tb testing.TB, ds *Datastore, fn func(q sqlx.ExtContext) error) { tb.Helper() err := fn(ds.primary) @@ -442,6 +650,22 @@ func ExecAdhocSQLWithError(ds *Datastore, fn func(q sqlx.ExtContext) error) erro return fn(ds.primary) } +// InsertAndGetLastID executes an INSERT statement and returns the auto-generated ID. +// On MySQL it uses LastInsertId(); on PG it appends RETURNING id and scans the result. +func InsertAndGetLastID(ctx context.Context, ds *Datastore, query string, args ...interface{}) (int64, error) { + if ds.dialect.IsPostgres() { + pgQuery := query + " RETURNING id" + var id int64 + err := sqlx.GetContext(ctx, ds.writer(ctx), &id, pgQuery, args...) + return id, err + } + result, err := ds.writer(ctx).ExecContext(ctx, query, args...) + if err != nil { + return 0, err + } + return result.LastInsertId() +} + // EncryptWithPrivateKey encrypts data with the server private key associated // with the Datastore. func EncryptWithPrivateKey(tb testing.TB, ds *Datastore, data []byte) ([]byte, error) { @@ -462,6 +686,40 @@ func TruncateTables(t testing.TB, ds *Datastore, tables ...string) { "osquery_options": true, "software_categories": true, } + + if _, ok := ds.dialect.(postgresDialect); ok { + db := ds.writer(context.Background()) + ctx := context.Background() + + // If no specific tables given, query all tables from PG catalog + if len(tables) == 0 { + rows, err := db.QueryContext(ctx, + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'") + if err != nil { + t.Logf("PG truncate: list tables: %v", err) + return + } + defer rows.Close() + for rows.Next() { + var tbl string + if err := rows.Scan(&tbl); err == nil { + tables = append(tables, tbl) + } + } + if err := rows.Err(); err != nil { + t.Logf("PG truncate: rows iteration: %v", err) + } + } + + for _, tbl := range tables { + if nonEmptyTables[tbl] { + continue + } + _, _ = db.ExecContext(ctx, `TRUNCATE TABLE "`+tbl+`" CASCADE`) + } + return + } + testing_utils.TruncateTables(t, ds.writer(context.Background()), ds.logger, nonEmptyTables, tables...) } @@ -571,7 +829,7 @@ func generateDummyWindowsProfileContents(uuid string) fleet.MDMWindowsProfileCon } func generateDummyWindowsProfile(uuid string) []byte { - return fmt.Appendf([]byte{}, `./Device/Foo/%s`, uuid) + return fmt.Appendf([]byte{}, `./Device/Foo/%s`, uuid) } // TODO(roberto): update when we have datastore functions and API methods for this From 045e0e1f35c42477c9403bdbea6c9dbbe7f9a9cd Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Wed, 1 Apr 2026 09:31:25 -0400 Subject: [PATCH 3/6] fix(datastore): PostgreSQL SQL compatibility across all datastore files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace MySQL-specific SQL constructs with dialect-abstracted equivalents throughout server/datastore/mysql/ and server/platform/mysql/: - INSERT IGNORE → ds.dialect.InsertIgnoreInto() + OnConflictDoNothing() - ON DUPLICATE KEY UPDATE → ds.dialect.OnDuplicateKey() - MySQL error codes (1062, 1292, etc.) → ds.dialect.IsDuplicate/IsDeadlock/etc. - Last-insert-id → insertAndGetID / insertAndGetIDTx (uses RETURNING on PG) - GROUP_CONCAT → ds.dialect.GroupConcat() - JSON_EXTRACT/JSON_ARRAYAGG → ds.dialect.JSONExtract/JSONArrayAgg() - TIMESTAMPDIFF / DATE_FORMAT → dialect helpers - Pagination SQL → ds.dialect.LimitOffset() - Bitwise coercions and BINARY casts → dialect helpers Also adds missing upstream methods and restores schema.sql to upstream state. --- server/datastore/mysql/activities.go | 36 +- server/datastore/mysql/aggregated_stats.go | 37 +- server/datastore/mysql/android.go | 115 ++-- server/datastore/mysql/android_enterprises.go | 3 +- server/datastore/mysql/android_hosts.go | 6 +- server/datastore/mysql/app_configs.go | 2 +- server/datastore/mysql/apple_mdm.go | 497 +++++++++------- server/datastore/mysql/ca_config_assets.go | 7 +- server/datastore/mysql/calendar_events.go | 27 +- server/datastore/mysql/campaigns.go | 6 +- server/datastore/mysql/carves.go | 10 +- .../mysql/certificate_authorities.go | 37 +- .../datastore/mysql/certificate_templates.go | 89 +-- .../mysql/conditional_access_bypass.go | 9 +- .../mysql/conditional_access_microsoft.go | 5 +- .../mysql/conditional_access_scep.go | 22 +- server/datastore/mysql/cron_stats.go | 18 +- server/datastore/mysql/delete.go | 2 +- server/datastore/mysql/disk_encryption.go | 4 +- server/datastore/mysql/errors.go | 8 + .../mysql/host_certificate_templates.go | 7 +- server/datastore/mysql/hosts.go | 544 ++++++++++++------ server/datastore/mysql/in_house_apps.go | 45 +- server/datastore/mysql/invites.go | 5 +- server/datastore/mysql/jobs.go | 3 +- server/datastore/mysql/labels.go | 86 ++- server/datastore/mysql/locks.go | 2 +- server/datastore/mysql/maintained_apps.go | 92 +-- server/datastore/mysql/mdm.go | 84 +-- server/datastore/mysql/microsoft_mdm.go | 4 +- server/datastore/mysql/nanomdm_storage.go | 20 +- .../mysql/operating_system_vulnerabilities.go | 52 +- server/datastore/mysql/operating_systems.go | 6 +- server/datastore/mysql/packs.go | 20 +- server/datastore/mysql/password_reset.go | 6 +- server/datastore/mysql/policies.go | 191 +++--- server/datastore/mysql/queries.go | 61 +- server/datastore/mysql/query_results.go | 6 +- server/datastore/mysql/scheduled_queries.go | 39 +- server/datastore/mysql/schema.sql | 10 +- server/datastore/mysql/scim.go | 18 +- server/datastore/mysql/scripts.go | 144 +++-- server/datastore/mysql/secret_variables.go | 5 +- server/datastore/mysql/sessions.go | 3 +- server/datastore/mysql/setup_experience.go | 17 +- server/datastore/mysql/software.go | 254 ++++---- server/datastore/mysql/software_installers.go | 111 ++-- .../mysql/software_title_display_names.go | 5 +- .../datastore/mysql/software_title_icons.go | 3 +- server/datastore/mysql/software_titles.go | 21 +- server/datastore/mysql/statistics.go | 3 +- server/datastore/mysql/teams.go | 43 +- server/datastore/mysql/users.go | 8 +- server/datastore/mysql/vpp.go | 83 ++- server/datastore/mysql/windows_updates.go | 2 +- server/datastore/mysql/wstep.go | 8 +- server/platform/endpointer/endpoint_utils.go | 2 - server/platform/mysql/common.go | 10 +- server/platform/mysql/list_options.go | 15 +- .../mysql/testing_utils/testing_utils.go | 43 +- 60 files changed, 1574 insertions(+), 1447 deletions(-) diff --git a/server/datastore/mysql/activities.go b/server/datastore/mysql/activities.go index 6d91765b75c..f8a678afd62 100644 --- a/server/datastore/mysql/activities.go +++ b/server/datastore/mysql/activities.go @@ -60,7 +60,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint 'script_name', COALESCE(ses.name, scr.name, ''), 'script_execution_id', ua.execution_id, 'batch_execution_id', bahr.batch_execution_id, - 'async', NOT ua.payload->'$.sync_request', + 'async', COALESCE(ua.payload->>'$.sync_request', '0') != '1', 'policy_id', sua.policy_id, 'policy_name', p.name ) as details, @@ -104,7 +104,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint 'software_package', COALESCE(si.filename, ua.payload->>'$.installer_filename', ''), 'install_uuid', ua.execution_id, 'status', 'pending_install', - 'self_service', ua.payload->'$.self_service' IS TRUE, + 'self_service', COALESCE(ua.payload->>'$.self_service', '0') = '1', 'source', COALESCE(st.source, ua.payload->>'$.source'), 'policy_id', siua.policy_id, 'policy_name', p.name @@ -146,7 +146,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint 'software_title', COALESCE(st.name, ua.payload->>'$.software_title_name', ''), 'script_execution_id', ua.execution_id, 'status', 'pending_uninstall', - 'self_service', COALESCE(ua.payload->'$.self_service', FALSE) IS TRUE, + 'self_service', COALESCE(ua.payload->>'$.self_service', '0') = '1', 'source', COALESCE(st.source, ua.payload->>'$.source'), 'policy_id', siua.policy_id, 'policy_name', p.name @@ -188,7 +188,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint 'software_title', COALESCE(st.name, ''), 'app_store_id', vaua.adam_id, 'command_uuid', ua.execution_id, - 'self_service', ua.payload->'$.self_service' IS TRUE, + 'self_service', COALESCE(ua.payload->>'$.self_service', '0') = '1', 'status', 'pending_install', 'host_platform', h.platform ) AS details, @@ -228,7 +228,7 @@ func (ds *Datastore) ListHostUpcomingActivities(ctx context.Context, hostID uint 'host_display_name', COALESCE(hdn.display_name, ''), 'software_title', COALESCE(st.name, ''), 'command_uuid', ua.execution_id, - 'self_service', ua.payload->'$.self_service' IS TRUE, + 'self_service', COALESCE(ua.payload->>'$.self_service', '0') = '1', 'status', 'pending_install' ) AS details, IF(ua.activated_at IS NULL, 0, 1) as topmost, @@ -809,10 +809,10 @@ func (ds *Datastore) GetHostUpcomingActivityMeta(ctx context.Context, hostID uin ua.activated_at, ua.activity_type, CASE - WHEN hma.lock_ref = :execution_id THEN :lock_action - WHEN hma.unlock_ref = :execution_id THEN :unlock_action - WHEN hma.wipe_ref = :execution_id THEN :wipe_action - ELSE :none_action + WHEN hma.lock_ref = :execution_id THEN CAST(:lock_action AS UNSIGNED) + WHEN hma.unlock_ref = :execution_id THEN CAST(:unlock_action AS UNSIGNED) + WHEN hma.wipe_ref = :execution_id THEN CAST(:wipe_action AS UNSIGNED) + ELSE CAST(:none_action AS UNSIGNED) END AS well_known_action FROM upcoming_activities ua @@ -1070,9 +1070,9 @@ SELECT sua.script_id, sua.policy_id, ua.user_id, - COALESCE(ua.payload->'$.sync_request', 0), + COALESCE(ua.payload->>'$.sync_request', '0') = '1', sua.setup_experience_script_id, - COALESCE(ua.payload->'$.is_internal', 0) + COALESCE(ua.payload->>'$.is_internal', '0') = '1' FROM upcoming_activities ua INNER JOIN script_upcoming_activities sua @@ -1108,7 +1108,7 @@ SELECT ua.host_id, siua.software_installer_id, ua.user_id, - COALESCE(ua.payload->'$.self_service', 0), + COALESCE(ua.payload->>'$.self_service', '0') = '1', siua.policy_id, COALESCE(si.filename, ua.payload->>'$.installer_filename', '[deleted installer]'), COALESCE(si.version, ua.payload->>'$.version', 'unknown'), @@ -1120,7 +1120,7 @@ SELECT -- the number of prior tries. +1 makes this the next attempt in sequence: -- first install = 1, first retry = 2, second retry = 3, etc. CASE - WHEN siua.policy_id IS NULL AND COALESCE(ua.payload->'$.with_retries', 0) = 1 THEN ( + WHEN siua.policy_id IS NULL AND COALESCE(ua.payload->>'$.with_retries', '0') = '1' THEN ( SELECT COUNT(*) + 1 FROM host_software_installs hsi2 WHERE hsi2.host_id = ua.host_id @@ -1171,7 +1171,7 @@ SELECT si.uninstall_script_content_id, '', ua.user_id, - 1 + TRUE FROM upcoming_activities ua INNER JOIN software_install_upcoming_activities siua @@ -1195,11 +1195,11 @@ SELECT ua.host_id, siua.software_installer_id, ua.user_id, - 1, -- uninstall + TRUE, -- uninstall '', -- no installer_filename for uninstalls COALESCE(si.title_id, siua.software_title_id), COALESCE(st.name, ua.payload->>'$.software_title_name', '[deleted title]'), - COALESCE(ua.payload->>'$.self_service', FALSE), + COALESCE(ua.payload->>'$.self_service', '0') = '1', 'unknown' FROM upcoming_activities ua @@ -1255,7 +1255,7 @@ SELECT ua.execution_id, ua.user_id, ua.payload->>'$.associated_event_id', - COALESCE(ua.payload->'$.self_service', 0), + COALESCE(ua.payload->>'$.self_service', '0') = '1', vaua.policy_id FROM upcoming_activities ua @@ -1291,7 +1291,7 @@ SELECT ua.execution_id, ua.user_id, iha.platform, - COALESCE(ua.payload->'$.self_service', 0) + COALESCE(ua.payload->>'$.self_service', '0') = '1' FROM upcoming_activities ua INNER JOIN in_house_app_upcoming_activities ihua diff --git a/server/datastore/mysql/aggregated_stats.go b/server/datastore/mysql/aggregated_stats.go index 21ffb74532d..69c42314e3b 100644 --- a/server/datastore/mysql/aggregated_stats.go +++ b/server/datastore/mysql/aggregated_stats.go @@ -44,24 +44,43 @@ FROM (SELECT (@rownum := @rownum + 1) AS row_number_value, sum1.* GROUP BY d.host_id) as sum2) AS t2 WHERE t1.row_number_value = FLOOR(total_rows * %[2]s) + 1` +const scheduledQueryPercentileQueryPG = ` +SELECT COALESCE((t1.%[1]s_total / t1.executions_total), 0) +FROM (SELECT ROW_NUMBER() OVER (ORDER BY (SUM(d.%[1]s) / SUM(d.executions))) AS row_number_value, + SUM(d.%[1]s) as %[1]s_total, SUM(d.executions) as executions_total + FROM scheduled_query_stats d + WHERE d.scheduled_query_id = ? + AND d.executions > 0 + GROUP BY d.host_id) AS t1, + (SELECT COUNT(*) AS total_rows + FROM (SELECT 1 + FROM scheduled_query_stats d + WHERE d.scheduled_query_id = ? + AND d.executions > 0 + GROUP BY d.host_id) as sum2) AS t2 +WHERE t1.row_number_value = FLOOR(total_rows * %[2]s) + 1` + const ( scheduledQueryTotalExecutions = `SELECT coalesce(sum(executions), 0) FROM scheduled_query_stats WHERE scheduled_query_id=?` ) -func getPercentileQuery(aggregate fleet.AggregatedStatsType, time string, percentile string) string { +func getPercentileQuery(aggregate fleet.AggregatedStatsType, time string, percentile string, isPG bool) string { switch aggregate { //nolint:gocritic // ignore singleCaseSwitch case fleet.AggregatedStatsTypeScheduledQuery: + if isPG { + return fmt.Sprintf(scheduledQueryPercentileQueryPG, time, percentile) + } return fmt.Sprintf(scheduledQueryPercentileQuery, time, percentile) } return "" } func setP50AndP95Map( - ctx context.Context, tx sqlx.QueryerContext, aggregate fleet.AggregatedStatsType, time string, id uint, statsMap map[string]interface{}, + ctx context.Context, tx sqlx.QueryerContext, aggregate fleet.AggregatedStatsType, time string, id uint, statsMap map[string]interface{}, isPG bool, ) error { var p50, p95 float64 - err := sqlx.GetContext(ctx, tx, &p50, getPercentileQuery(aggregate, time, "0.5"), id, id) + err := sqlx.GetContext(ctx, tx, &p50, getPercentileQuery(aggregate, time, "0.5", isPG), id, id) if err != nil { if err == sql.ErrNoRows { return nil @@ -69,7 +88,7 @@ func setP50AndP95Map( return ctxerr.Wrapf(ctx, err, "getting %s p50 for %s %d", time, aggregate, id) } statsMap[time+"_p50"] = p50 - err = sqlx.GetContext(ctx, tx, &p95, getPercentileQuery(aggregate, time, "0.95"), id, id) + err = sqlx.GetContext(ctx, tx, &p95, getPercentileQuery(aggregate, time, "0.95", isPG), id, id) if err != nil { if err == sql.ErrNoRows { return nil @@ -103,10 +122,11 @@ func (ds *Datastore) CalculateAggregatedPerfStatsPercentiles(ctx context.Context // many queries is not ideal, but getting both values and totals in the same query was a bit more complicated // so I went for the simpler approach first, we can optimize later - if err := setP50AndP95Map(ctx, reader, aggregate, "user_time", queryID, statsMap); err != nil { + _, isPG := ds.dialect.(postgresDialect) + if err := setP50AndP95Map(ctx, reader, aggregate, "user_time", queryID, statsMap, isPG); err != nil { return err } - if err := setP50AndP95Map(ctx, reader, aggregate, "system_time", queryID, statsMap); err != nil { + if err := setP50AndP95Map(ctx, reader, aggregate, "system_time", queryID, statsMap, isPG); err != nil { return err } @@ -128,9 +148,8 @@ func (ds *Datastore) CalculateAggregatedPerfStatsPercentiles(ctx context.Context ctx, ` INSERT INTO aggregated_stats(id, type, global_stats, json_value) - VALUES (?, ?, 0, ?) - ON DUPLICATE KEY UPDATE json_value=VALUES(json_value) - `, + VALUES (?, ?, false, ?) + `+ds.dialect.OnDuplicateKey("id,type,global_stats", `json_value=VALUES(json_value)`), queryID, aggregate, statsJson, ) if err != nil { diff --git a/server/datastore/mysql/android.go b/server/datastore/mysql/android.go index fc69b463c9e..798d1ecc029 100644 --- a/server/datastore/mysql/android.go +++ b/server/datastore/mysql/android.go @@ -47,23 +47,8 @@ func (ds *Datastore) NewAndroidHost(ctx context.Context, host *fleet.AndroidHost detail_updated_at, label_updated_at, uuid - ) VALUES ( - :node_key, - :hostname, - :computer_name, - :platform, - :os_version, - :build, - :memory, - :team_id, - :hardware_serial, - :cpu_type, - :hardware_model, - :hardware_vendor, - :detail_updated_at, - :label_updated_at, - :uuid - ) ON DUPLICATE KEY UPDATE + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ` + ds.dialect.OnDuplicateKey("node_key", ` hostname = VALUES(hostname), computer_name = VALUES(computer_name), platform = VALUES(platform), @@ -78,28 +63,27 @@ func (ds *Datastore) NewAndroidHost(ctx context.Context, host *fleet.AndroidHost detail_updated_at = VALUES(detail_updated_at), label_updated_at = VALUES(label_updated_at), uuid = VALUES(uuid) - ` - result, err := sqlx.NamedExecContext(ctx, tx, stmt, map[string]interface{}{ - "node_key": host.NodeKey, - "hostname": host.Hostname, - "computer_name": host.ComputerName, - "platform": host.Platform, - "os_version": host.OSVersion, - "build": host.Build, - "memory": host.Memory, - "team_id": host.TeamID, - "hardware_serial": host.HardwareSerial, - "cpu_type": host.CPUType, - "hardware_model": host.HardwareModel, - "hardware_vendor": host.HardwareVendor, - "detail_updated_at": host.DetailUpdatedAt, - "label_updated_at": host.LabelUpdatedAt, - "uuid": host.UUID, - }) + `) + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, stmt, + host.NodeKey, + host.Hostname, + host.ComputerName, + host.Platform, + host.OSVersion, + host.Build, + host.Memory, + host.TeamID, + host.HardwareSerial, + host.CPUType, + host.HardwareModel, + host.HardwareVendor, + host.DetailUpdatedAt, + host.LabelUpdatedAt, + host.UUID, + ) if err != nil { return ctxerr.Wrap(ctx, err, "new Android host") } - id, _ := result.LastInsertId() if id == 0 { // This was an UPDATE, not an INSERT, so we need to get the host ID var hostID uint @@ -113,7 +97,7 @@ func (ds *Datastore) NewAndroidHost(ctx context.Context, host *fleet.AndroidHost } host.Device.HostID = host.Host.ID - err = upsertHostDisplayNames(ctx, tx, *host.Host) + err = upsertHostDisplayNames(ctx, tx, ds.dialect, *host.Host) if err != nil { return ctxerr.Wrap(ctx, err, "new Android host display name") } @@ -124,7 +108,7 @@ func (ds *Datastore) NewAndroidHost(ctx context.Context, host *fleet.AndroidHost // create entry in host_mdm as enrolled (manually), because currently all // android hosts are necessarily MDM-enrolled when created. - if err := upsertAndroidHostMDMInfoDB(ctx, tx, appCfg.ServerSettings.ServerURL, companyOwned, true, host.Host.ID); err != nil { + if err := upsertAndroidHostMDMInfoDB(ctx, tx, ds.dialect, appCfg.ServerSettings.ServerURL, companyOwned, true, host.Host.ID); err != nil { return ctxerr.Wrap(ctx, err, "new Android host MDM info") } @@ -217,7 +201,7 @@ func (ds *Datastore) UpdateAndroidHost(ctx context.Context, host *fleet.AndroidH if fromEnroll { // update host_mdm to set enrolled back to true - if err := upsertAndroidHostMDMInfoDB(ctx, tx, appCfg.ServerSettings.ServerURL, companyOwned, true, host.Host.ID); err != nil { + if err := upsertAndroidHostMDMInfoDB(ctx, tx, ds.dialect, appCfg.ServerSettings.ServerURL, companyOwned, true, host.Host.ID); err != nil { return ctxerr.Wrap(ctx, err, "update Android host MDM info") } @@ -367,7 +351,7 @@ func (ds *Datastore) insertAndroidHostLabelMembershipTx(ctx context.Context, tx _, err = tx.ExecContext(ctx, ` INSERT INTO label_membership (host_id, label_id) VALUES (?, ?), (?, ?) - ON DUPLICATE KEY UPDATE host_id = host_id`, + `+ds.dialect.OnDuplicateKey("host_id,label_id", `host_id = VALUES(host_id)`), hostID, allHostsLabelID, hostID, androidLabelID) if err != nil { return ctxerr.Wrap(ctx, err, "set label membership") @@ -429,19 +413,17 @@ UPDATE host_mdm return rows > 0, nil } -func upsertAndroidHostMDMInfoDB(ctx context.Context, tx sqlx.ExtContext, serverURL string, companyOwned, enrolled bool, hostID uint) error { - result, err := tx.ExecContext(ctx, ` +func upsertAndroidHostMDMInfoDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, serverURL string, companyOwned, enrolled bool, hostID uint) error { + mdmID, err := insertAndGetIDTx(ctx, tx, dialect, ` INSERT INTO mobile_device_management_solutions (name, server_url) VALUES (?, ?) - ON DUPLICATE KEY UPDATE server_url = VALUES(server_url)`, + `+dialect.OnDuplicateKey("name, server_url", "server_url = VALUES(server_url)"), fleet.WellKnownMDMFleet, serverURL) if err != nil { return ctxerr.Wrap(ctx, err, "upsert mdm solution") } - - var mdmID int64 - if insertOnDuplicateDidInsertOrUpdate(result) { - mdmID, _ = result.LastInsertId() - } else { + if mdmID == 0 { + // ON DUPLICATE KEY UPDATE did not insert a new row (MySQL returns 0 for LastInsertId); + // fall back to querying the existing row's ID. stmt := `SELECT id FROM mobile_device_management_solutions WHERE name = ? AND server_url = ?` if err := sqlx.GetContext(ctx, tx, &mdmID, stmt, fleet.WellKnownMDMFleet, serverURL); err != nil { return ctxerr.Wrap(ctx, err, "query mdm solution id") @@ -455,7 +437,7 @@ func upsertAndroidHostMDMInfoDB(ctx context.Context, tx sqlx.ExtContext, serverU _, err = tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO host_mdm (enrolled, server_url, installed_from_dep, mdm_id, is_server, is_personal_enrollment, host_id) VALUES %s - ON DUPLICATE KEY UPDATE enrolled = VALUES(enrolled), server_url = VALUES(server_url), mdm_id = VALUES(mdm_id), is_personal_enrollment = VALUES(is_personal_enrollment)`, strings.Join(parts, ",")), args...) + `+dialect.OnDuplicateKey("host_id", "enrolled = VALUES(enrolled), server_url = VALUES(server_url), mdm_id = VALUES(mdm_id), is_personal_enrollment = VALUES(is_personal_enrollment)"), strings.Join(parts, ",")), args...) return ctxerr.Wrap(ctx, err, "upsert host mdm info") } @@ -484,7 +466,7 @@ INSERT INTO res, err := tx.ExecContext(ctx, insertProfileStmt, profileUUID, teamID, cp.Name, cp.RawJSON, cp.Name, teamID, cp.Name, teamID, cp.Name, teamID) if err != nil { switch { - case IsDuplicate(err): + case ds.dialect.IsDuplicate(err): return &existsError{ ResourceType: "MDMAndroidConfigProfile.Name", Identifier: cp.Name, @@ -527,7 +509,7 @@ INSERT INTO if len(labels) == 0 { profsWithoutLabel = append(profsWithoutLabel, profileUUID) } - if _, err := batchSetProfileLabelAssociationsDB(ctx, tx, labels, profsWithoutLabel, "android"); err != nil { + if _, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, labels, profsWithoutLabel, "android"); err != nil { return ctxerr.Wrap(ctx, err, "inserting android profile label associations") } @@ -601,6 +583,7 @@ func (ds *Datastore) DeleteMDMAndroidConfigProfile(ctx context.Context, profileU func (ds *Datastore) GetMDMAndroidProfilesSummary(ctx context.Context, teamID *uint) (*fleet.MDMProfilesSummary, error) { stmt := ` +SELECT count, status FROM ( SELECT COUNT(id) AS count, %s AS status @@ -613,7 +596,8 @@ WHERE hmdm.enrolled = 1 AND %s GROUP BY - status HAVING status IS NOT NULL` + status +) sq WHERE status IS NOT NULL` teamFilter := "team_id IS NULL" if teamID != nil && *teamID > 0 { @@ -705,7 +689,7 @@ func sqlJoinMDMAndroidProfilesStatus() string { host_uuid, IF(status IS NULL OR status IN (` + certPending + `, ` + certDelivering + `, ` + certDelivered + `), 1, 0) AS prof_pending, IF(status = ` + certFailed + `, 1, 0) AS prof_failed, - 0 AS prof_verifying, + IF(1=0, 1, 0) AS prof_verifying, IF(status = ` + certVerified + ` AND operation_type = ` + install + `, 1, 0) AS prof_verified FROM host_certificate_templates @@ -849,7 +833,7 @@ const androidApplicableProfilesQuery = ` GROUP BY macp.profile_uuid, macp.name, h.uuid, h.id HAVING - count_profile_labels > 0 AND count_host_labels = count_profile_labels + COUNT(*) > 0 AND COUNT(lm.label_id) = COUNT(*) UNION @@ -894,8 +878,11 @@ const androidApplicableProfilesQuery = ` HAVING -- considers only the profiles with labels, without any broken label, with results reported after all labels were -- created and with the host not in any label - count_profile_labels > 0 AND count_profile_labels = count_non_broken_labels AND - count_profile_labels = count_host_updated_after_labels AND count_host_labels = 0 + COUNT(*) > 0 AND COUNT(*) = COUNT(mcpl.label_id) AND + COUNT(*) = SUM( + CASE WHEN lbl.label_membership_type <> 1 AND lbl.created_at IS NOT NULL AND h.label_updated_at >= lbl.created_at THEN 1 + WHEN lbl.label_membership_type = 1 AND lbl.created_at IS NOT NULL THEN 1 + ELSE 0 END) AND COUNT(lm.label_id) = 0 UNION @@ -927,7 +914,7 @@ const androidApplicableProfilesQuery = ` GROUP BY macp.profile_uuid, macp.name, h.uuid, h.id HAVING - count_profile_labels > 0 AND count_host_labels >= 1 + COUNT(*) > 0 AND COUNT(lm.label_id) >= 1 ` // ListMDMAndroidProfilesToSend is the android platform equivalent to @@ -1139,7 +1126,7 @@ func (ds *Datastore) BulkUpsertMDMAndroidHostProfiles(ctx context.Context, paylo can_reverify ) VALUES %s - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("host_uuid,profile_uuid", ` status = VALUES(status), operation_type = VALUES(operation_type), detail = VALUES(detail), @@ -1149,7 +1136,7 @@ func (ds *Datastore) BulkUpsertMDMAndroidHostProfiles(ctx context.Context, paylo request_fail_count = VALUES(request_fail_count), included_in_policy_version = VALUES(included_in_policy_version), can_reverify = VALUES(can_reverify) -`, strings.TrimSuffix(valuePart, ","), +`), strings.TrimSuffix(valuePart, ","), ) // Taken from BulkUpsertMDMAppleHostProfiles: We need to run with retry @@ -1355,7 +1342,7 @@ WHERE } // Insert or update incoming profiles - const insertNewOrEditedProfile = ` + insertNewOrEditedProfile := ` INSERT INTO mdm_android_configuration_profiles ( profile_uuid, team_id, @@ -1363,11 +1350,11 @@ WHERE raw_json, uploaded_at ) VALUES (CONCAT('` + fleet.MDMAndroidProfileUUIDPrefix + `', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, CURRENT_TIMESTAMP(6)) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("profile_uuid", ` raw_json = VALUES(raw_json), name = VALUES(name), uploaded_at = IF(raw_json = VALUES(raw_json) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP(6)) -` +`) for _, p := range profiles { var res sql.Result if res, err = tx.ExecContext(ctx, insertNewOrEditedProfile, profileTeamID, p.Name, p.RawJSON); err != nil { @@ -1815,9 +1802,9 @@ func (ds *Datastore) updateAndroidAppConfigurationTx(ctx context.Context, tx sql INSERT INTO android_app_configurations (application_id, team_id, global_or_team_id, configuration) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("global_or_team_id,application_id", ` configuration = VALUES(configuration) - ` + `) _, err = tx.ExecContext(ctx, stmt, appID, ptr.UintOrNilIfZero(teamID), teamID, config) if err != nil { diff --git a/server/datastore/mysql/android_enterprises.go b/server/datastore/mysql/android_enterprises.go index d26145f555d..857efb9b913 100644 --- a/server/datastore/mysql/android_enterprises.go +++ b/server/datastore/mysql/android_enterprises.go @@ -14,11 +14,10 @@ import ( func (ds *AndroidDatastore) CreateEnterprise(ctx context.Context, userID uint) (uint, error) { // android_enterprises user_id is only set when the row is created stmt := `INSERT INTO android_enterprises (signup_name, user_id) VALUES ('', ?)` - res, err := ds.Writer(ctx).ExecContext(ctx, stmt, userID) + id, err := insertAndGetIDTx(ctx, ds.Writer(ctx), ds.dialect, stmt, userID) if err != nil { return 0, ctxerr.Wrap(ctx, err, "inserting enterprise") } - id, _ := res.LastInsertId() return uint(id), nil // nolint:gosec // dismiss G115 } diff --git a/server/datastore/mysql/android_hosts.go b/server/datastore/mysql/android_hosts.go index 7e8998006f8..9486a47c27e 100644 --- a/server/datastore/mysql/android_hosts.go +++ b/server/datastore/mysql/android_hosts.go @@ -66,7 +66,7 @@ func (ds *AndroidDatastore) insertDevice(ctx context.Context, device *android.De applied_policy_version ) VALUES (?, ?, ?, ?, ?, ?)` - result, err := tx.ExecContext(ctx, stmt, + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, stmt, device.HostID, device.DeviceID, device.EnterpriseSpecificID, @@ -77,10 +77,6 @@ func (ds *AndroidDatastore) insertDevice(ctx context.Context, device *android.De if err != nil { return nil, ctxerr.Wrap(ctx, err, "inserting device") } - id, err := result.LastInsertId() - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "getting android_devices last insert ID") - } device.ID = uint(id) // nolint:gosec return device, nil } diff --git a/server/datastore/mysql/app_configs.go b/server/datastore/mysql/app_configs.go index 76ffeaede48..771f93cf85c 100644 --- a/server/datastore/mysql/app_configs.go +++ b/server/datastore/mysql/app_configs.go @@ -67,7 +67,7 @@ func (ds *Datastore) SaveAppConfig(ctx context.Context, info *fleet.AppConfig) e } _, err = tx.ExecContext(ctx, - `INSERT INTO app_config_json(json_value) VALUES(?) ON DUPLICATE KEY UPDATE json_value = VALUES(json_value)`, + `INSERT INTO app_config_json(json_value) VALUES(?) `+ds.dialect.OnDuplicateKey("id", `json_value = VALUES(json_value)`), configBytes, ) if err != nil { diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index 16f52edfd62..53382a28893 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -212,30 +212,55 @@ INSERT INTO if err != nil { return err } - res, err := tx.ExecContext(ctx, stmt, - profUUID, teamID, cp.Identifier, cp.Name, cp.Scope, cp.Mobileconfig, cp.Mobileconfig, cp.SecretsUpdatedAt, cp.Name, teamID, cp.Name, - teamID, cp.Name, teamID) - if err != nil { - switch { - case IsDuplicate(err): - return ctxerr.Wrap(ctx, formatErrorDuplicateConfigProfile(err, &cp)) - default: - return ctxerr.Wrap(ctx, err, "creating new apple mdm config profile") + if ds.dialect.ReturningID() != "" { + // PostgreSQL: RETURNING profile_id (this table uses profile_id, not id) + err := tx.QueryRowxContext(ctx, stmt+" RETURNING profile_id", + profUUID, teamID, cp.Identifier, cp.Name, cp.Scope, cp.Mobileconfig, cp.Mobileconfig, cp.SecretsUpdatedAt, cp.Name, teamID, cp.Name, + teamID, cp.Name, teamID).Scan(&profileID) + if errors.Is(err, sql.ErrNoRows) { + return &existsError{ + ResourceType: "MDMAppleConfigProfile.PayloadDisplayName", + Identifier: cp.Name, + TeamID: cp.TeamID, + } + } else if err != nil { + switch { + case ds.dialect.IsDuplicate(err): + return ctxerr.Wrap(ctx, formatErrorDuplicateConfigProfile(err, &cp)) + default: + return ctxerr.Wrap(ctx, err, "creating new apple mdm config profile") + } + } + } else { + res, err := tx.ExecContext(ctx, stmt, + profUUID, teamID, cp.Identifier, cp.Name, cp.Scope, cp.Mobileconfig, cp.Mobileconfig, cp.SecretsUpdatedAt, cp.Name, teamID, cp.Name, + teamID, cp.Name, teamID) + if err != nil { + switch { + case ds.dialect.IsDuplicate(err): + return ctxerr.Wrap(ctx, formatErrorDuplicateConfigProfile(err, &cp)) + default: + return ctxerr.Wrap(ctx, err, "creating new apple mdm config profile") + } } - } - aff, _ := res.RowsAffected() - if aff == 0 { - return &existsError{ - ResourceType: "MDMAppleConfigProfile.PayloadDisplayName", - Identifier: cp.Name, - TeamID: cp.TeamID, + aff, _ := res.RowsAffected() + if aff == 0 { + return &existsError{ + ResourceType: "MDMAppleConfigProfile.PayloadDisplayName", + Identifier: cp.Name, + TeamID: cp.TeamID, + } } - } - // record the ID as we want to return a fleet.Profile instance with it - // filled in. - profileID, _ = res.LastInsertId() + // record the ID as we want to return a fleet.Profile instance with it + // filled in. + profileID, _ = res.LastInsertId() // PG: returns 0 + if profileID == 0 { + // Fallback for PG: get the ID by profile_uuid + _ = sqlx.GetContext(ctx, tx, &profileID, `SELECT profile_id FROM mdm_apple_configuration_profiles WHERE profile_uuid = ?`, profUUID) + } + } labels := make([]fleet.ConfigurationProfileLabel, 0, len(cp.LabelsIncludeAll)+len(cp.LabelsIncludeAny)+len(cp.LabelsExcludeAny)) for i := range cp.LabelsIncludeAll { @@ -260,10 +285,10 @@ INSERT INTO if len(labels) == 0 { profWithoutLabels = append(profWithoutLabels, profUUID) } - if _, err := batchSetProfileLabelAssociationsDB(ctx, tx, labels, profWithoutLabels, "darwin"); err != nil { + if _, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, labels, profWithoutLabels, "darwin"); err != nil { return ctxerr.Wrap(ctx, err, "inserting darwin profile label associations") } - if _, err := batchSetProfileVariableAssociationsDB(ctx, tx, []fleet.MDMProfileUUIDFleetVariables{ + if _, err := batchSetProfileVariableAssociationsDB(ctx, tx, ds.dialect, []fleet.MDMProfileUUIDFleetVariables{ {ProfileUUID: profUUID, FleetVariables: usesFleetVars}, }, "darwin"); err != nil { return ctxerr.Wrap(ctx, err, "inserting darwin profile variable associations") @@ -447,7 +472,7 @@ SELECT name, identifier, raw_json, - token, + COALESCE(token, '') AS token, created_at, uploaded_at, secrets_updated_at @@ -616,7 +641,7 @@ func cancelAppleHostInstallsForDeletedMDMProfiles(ctx context.Context, tx sqlx.E host_mdm_apple_profiles SET operation_type = ?, - ignore_error = IF(status IN (?), 1, 0), + ignore_error = IF(status IN (?), TRUE, FALSE), status = NULL WHERE profile_uuid IN (?) AND @@ -759,7 +784,7 @@ SELECT COALESCE(detail, '') AS detail, scope, CASE - WHEN scope = 'user' THEN COALESCE((SELECT nu.user_short_name FROM nano_enrollments ne INNER JOIN nano_users nu ON ne.user_id = nu.id WHERE ne.type = 'User' AND ne.enabled = 1 AND ne.device_id = host_uuid ORDER BY ne.created_at ASC LIMIT 1), '') + WHEN scope = 'User' THEN COALESCE((SELECT nu.user_short_name FROM nano_enrollments ne INNER JOIN nano_users nu ON ne.user_id = nu.id WHERE ne.type = 'User' AND ne.enabled = 1 AND ne.device_id = host_uuid ORDER BY ne.created_at ASC LIMIT 1), '') ELSE '' END AS managed_local_account FROM @@ -907,22 +932,21 @@ func (ds *Datastore) NewMDMAppleEnrollmentProfile( ctx context.Context, payload fleet.MDMAppleEnrollmentProfilePayload, ) (*fleet.MDMAppleEnrollmentProfile, error) { - res, err := ds.writer(ctx).ExecContext(ctx, + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), ` INSERT INTO mdm_apple_enrollment_profiles (token, type, dep_profile) VALUES (?, ?, ?) -ON DUPLICATE KEY UPDATE +`+ds.dialect.OnDuplicateKey("id", ` token = VALUES(token), type = VALUES(type), dep_profile = VALUES(dep_profile) -`, +`), payload.Token, payload.Type, payload.DEPProfile, ) if err != nil { return nil, ctxerr.Wrap(ctx, err) } - id, _ := res.LastInsertId() return &fleet.MDMAppleEnrollmentProfile{ ID: uint(id), //nolint:gosec // dismiss G115 Token: payload.Token, @@ -1155,15 +1179,13 @@ WHERE } func (ds *Datastore) NewMDMAppleInstaller(ctx context.Context, name string, size int64, manifest string, installer []byte, urlToken string) (*fleet.MDMAppleInstaller, error) { - res, err := ds.writer(ctx).ExecContext( - ctx, + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), `INSERT INTO mdm_apple_installers (name, size, manifest, installer, url_token) VALUES (?, ?, ?, ?, ?)`, name, size, manifest, installer, urlToken, ) if err != nil { return nil, ctxerr.Wrap(ctx, err) } - id, _ := res.LastInsertId() return &fleet.MDMAppleInstaller{ ID: uint(id), //nolint:gosec // dismiss G115 Size: size, @@ -1272,13 +1294,14 @@ func (ds *Datastore) MDMAppleUpsertHost(ctx context.Context, mdmHost *fleet.Host return ctxerr.Wrap(ctx, err, "mdm apple upsert host get app config") } return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return ingestMDMAppleDeviceFromCheckinDB(ctx, tx, mdmHost, ds.logger, appCfg, fromPersonalEnrollment) + return ingestMDMAppleDeviceFromCheckinDB(ctx, tx, ds.dialect, mdmHost, ds.logger, appCfg, fromPersonalEnrollment) }) } func ingestMDMAppleDeviceFromCheckinDB( ctx context.Context, tx sqlx.ExtContext, + dialect DialectHelper, mdmHost *fleet.Host, logger *slog.Logger, appCfg *fleet.AppConfig, @@ -1296,13 +1319,13 @@ func ingestMDMAppleDeviceFromCheckinDB( enrolledHostInfo, err := matchHostDuringEnrollment(ctx, tx, mdmEnroll, true, "", mdmHost.UUID, mdmHost.HardwareSerial) switch { case errors.Is(err, sql.ErrNoRows): - return insertMDMAppleHostDB(ctx, tx, mdmHost, logger, appCfg, fromPersonalEnrollment) + return insertMDMAppleHostDB(ctx, tx, dialect, mdmHost, logger, appCfg, fromPersonalEnrollment) case err != nil: return ctxerr.Wrap(ctx, err, "get mdm apple host by serial number or udid") default: - return updateMDMAppleHostDB(ctx, tx, enrolledHostInfo.ID, mdmHost, appCfg, fromPersonalEnrollment) + return updateMDMAppleHostDB(ctx, tx, dialect, enrolledHostInfo.ID, mdmHost, appCfg, fromPersonalEnrollment) } } @@ -1322,6 +1345,7 @@ func mdmHostEnrollFields(mdmHost *fleet.Host) (refetchRequested bool, lastEnroll func updateMDMAppleHostDB( ctx context.Context, tx sqlx.ExtContext, + dialect DialectHelper, hostID uint, mdmHost *fleet.Host, appCfg *fleet.AppConfig, @@ -1376,7 +1400,7 @@ func updateMDMAppleHostDB( return ctxerr.Wrap(ctx, err, "error clearing mdm apple host_mdm_actions") } - if err := upsertMDMAppleHostMDMInfoDB(ctx, tx, appCfg, false, fromPersonalEnrollment, hostID); err != nil { + if err := upsertMDMAppleHostMDMInfoDB(ctx, tx, dialect, appCfg, false, fromPersonalEnrollment, hostID); err != nil { return ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert MDM info") } @@ -1386,6 +1410,7 @@ func updateMDMAppleHostDB( func insertMDMAppleHostDB( ctx context.Context, tx sqlx.ExtContext, + dialect DialectHelper, mdmHost *fleet.Host, logger *slog.Logger, appCfg *fleet.AppConfig, @@ -1404,8 +1429,10 @@ func insertMDMAppleHostDB( refetch_requested ) VALUES (?,?,?,?,?,?,?,?)` - res, err := tx.ExecContext( + id, err := insertAndGetIDTx( ctx, + tx, + dialect, insertStmt, mdmHost.HardwareSerial, mdmHost.UUID, @@ -1419,26 +1446,21 @@ func insertMDMAppleHostDB( if err != nil { return ctxerr.Wrap(ctx, err, "insert mdm apple host") } - - id, err := res.LastInsertId() - if err != nil { - return ctxerr.Wrap(ctx, err, "last insert id mdm apple host") - } if id < 1 { - return ctxerr.Wrap(ctx, err, "ingest mdm apple host unexpected last insert id") + return ctxerr.New(ctx, "ingest mdm apple host unexpected last insert id") } mdmHost.ID = uint(id) - if err := upsertHostDisplayNames(ctx, tx, *mdmHost); err != nil { + if err := upsertHostDisplayNames(ctx, tx, dialect, *mdmHost); err != nil { return ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert display names") } - if err := upsertMDMAppleHostLabelMembershipDB(ctx, tx, logger, *mdmHost); err != nil { + if err := upsertMDMAppleHostLabelMembershipDB(ctx, tx, dialect, logger, *mdmHost); err != nil { return ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert label membership") } - if err := upsertMDMAppleHostMDMInfoDB(ctx, tx, appCfg, false, fromPersonalEnrollment, mdmHost.ID); err != nil { + if err := upsertMDMAppleHostMDMInfoDB(ctx, tx, dialect, appCfg, false, fromPersonalEnrollment, mdmHost.ID); err != nil { return ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert MDM info") } return nil @@ -1466,6 +1488,7 @@ type hostToCreateFromMDM struct { func createHostFromMDMDB( ctx context.Context, tx sqlx.ExtContext, + dialect DialectHelper, logger *slog.Logger, devices []hostToCreateFromMDM, fromADE bool, @@ -1494,11 +1517,11 @@ func createHostFromMDMDB( '`+server.NeverTimestamp+`' AS last_enrolled_at, '`+server.NeverTimestamp+`' AS detail_updated_at, NULL AS osquery_host_id, - IF(us.platform = 'ios' OR us.platform = 'ipados', 0, 1) AS refetch_requested, + IF(us.platform = 'ios' OR us.platform = 'ipados', FALSE, TRUE) AS refetch_requested, CASE - WHEN us.platform = 'ios' THEN ? - WHEN us.platform = 'ipados' THEN ? - ELSE ? + WHEN us.platform = 'ios' THEN CAST(? AS UNSIGNED) + WHEN us.platform = 'ipados' THEN CAST(? AS UNSIGNED) + ELSE CAST(? AS UNSIGNED) END AS team_id FROM (%s) us LEFT JOIN hosts h ON us.hardware_serial = h.hardware_serial @@ -1581,11 +1604,11 @@ func createHostFromMDMDB( } } - if err := upsertHostDisplayNames(ctx, tx, hosts...); err != nil { + if err := upsertHostDisplayNames(ctx, tx, dialect, hosts...); err != nil { return 0, nil, ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert display names") } - if err := upsertMDMAppleHostLabelMembershipDB(ctx, tx, logger, hosts...); err != nil { + if err := upsertMDMAppleHostLabelMembershipDB(ctx, tx, dialect, logger, hosts...); err != nil { return 0, nil, ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert label membership") } @@ -1604,6 +1627,7 @@ func createHostFromMDMDB( if err := upsertMDMAppleHostMDMInfoDB( ctx, tx, + dialect, appCfg, fromADE, false, @@ -1630,7 +1654,7 @@ func (ds *Datastore) IngestMDMAppleDeviceFromOTAEnrollment( UUID: &deviceInfo.UDID, }, } - _, hosts, err := createHostFromMDMDB(ctx, tx, ds.logger, toInsert, false, teamID, teamID, teamID) + _, hosts, err := createHostFromMDMDB(ctx, tx, ds.dialect, ds.logger, toInsert, false, teamID, teamID, teamID) if idpUUID != "" && len(hosts) > 0 { host := hosts[0] ds.logger.InfoContext(ctx, fmt.Sprintf("associating host %s with idp account %s", host.UUID, idpUUID)) @@ -1697,6 +1721,7 @@ func (ds *Datastore) IngestMDMAppleDevicesFromDEPSync( n, hosts, err := createHostFromMDMDB( ctx, tx, + ds.dialect, ds.logger, htc, true, @@ -1767,7 +1792,7 @@ func upsertHostDEPAssignmentsDB(ctx context.Context, tx sqlx.ExtContext, hosts [ return nil } -func upsertHostDisplayNames(ctx context.Context, tx sqlx.ExtContext, hosts ...fleet.Host) error { +func upsertHostDisplayNames(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, hosts ...fleet.Host) error { var args []interface{} var parts []string for _, h := range hosts { @@ -1777,7 +1802,7 @@ func upsertHostDisplayNames(ctx context.Context, tx sqlx.ExtContext, hosts ...fl _, err := tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO host_display_names (host_id, display_name) VALUES %s - ON DUPLICATE KEY UPDATE display_name = VALUES(display_name)`, strings.Join(parts, ",")), + `+dialect.OnDuplicateKey("host_id", `display_name = VALUES(display_name)`), strings.Join(parts, ",")), args...) if err != nil { return ctxerr.Wrap(ctx, err, "upsert host display names") @@ -1786,7 +1811,7 @@ func upsertHostDisplayNames(ctx context.Context, tx sqlx.ExtContext, hosts ...fl return nil } -func upsertMDMAppleHostMDMInfoDB(ctx context.Context, tx sqlx.ExtContext, appCfg *fleet.AppConfig, fromSync, fromPersonalEnrollment bool, hostIDs ...uint) error { +func upsertMDMAppleHostMDMInfoDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, appCfg *fleet.AppConfig, fromSync, fromPersonalEnrollment bool, hostIDs ...uint) error { if len(hostIDs) == 0 { return nil } @@ -1800,18 +1825,16 @@ func upsertMDMAppleHostMDMInfoDB(ctx context.Context, tx sqlx.ExtContext, appCfg // enrolled yet. enrolled := !fromSync - result, err := tx.ExecContext(ctx, ` + mdmID, err := insertAndGetIDTx(ctx, tx, dialect, ` INSERT INTO mobile_device_management_solutions (name, server_url) VALUES (?, ?) - ON DUPLICATE KEY UPDATE server_url = VALUES(server_url)`, + `+dialect.OnDuplicateKey("name, server_url", "server_url = VALUES(server_url)"), fleet.WellKnownMDMFleet, serverURL) if err != nil { return ctxerr.Wrap(ctx, err, "upsert mdm solution") } - - var mdmID int64 - if insertOnDuplicateDidInsertOrUpdate(result) { - mdmID, _ = result.LastInsertId() - } else { + if mdmID == 0 { + // ON DUPLICATE KEY UPDATE did not insert a new row (MySQL returns 0 for LastInsertId); + // fall back to querying the existing row's ID. stmt := `SELECT id FROM mobile_device_management_solutions WHERE name = ? AND server_url = ?` if err := sqlx.GetContext(ctx, tx, &mdmID, stmt, fleet.WellKnownMDMFleet, serverURL); err != nil { return ctxerr.Wrap(ctx, err, "query mdm solution id") @@ -1827,12 +1850,12 @@ func upsertMDMAppleHostMDMInfoDB(ctx context.Context, tx sqlx.ExtContext, appCfg _, err = tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO host_mdm (enrolled, server_url, installed_from_dep, mdm_id, is_server, host_id, is_personal_enrollment) VALUES %s - ON DUPLICATE KEY UPDATE enrolled = VALUES(enrolled)`, strings.Join(parts, ",")), args...) + `+dialect.OnDuplicateKey("host_id", "enrolled = VALUES(enrolled)"), strings.Join(parts, ",")), args...) return ctxerr.Wrap(ctx, err, "upsert host mdm info") } -func upsertMDMAppleHostLabelMembershipDB(ctx context.Context, tx sqlx.ExtContext, logger *slog.Logger, hosts ...fleet.Host) error { +func upsertMDMAppleHostLabelMembershipDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, logger *slog.Logger, hosts ...fleet.Host) error { // Builtin label memberships are usually inserted when the first distributed // query results are received; however, we want to insert pending MDM hosts // now because it may still be some time before osquery is running on these @@ -1892,7 +1915,7 @@ func upsertMDMAppleHostLabelMembershipDB(ctx context.Context, tx sqlx.ExtContext } _, err = tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO label_membership (host_id, label_id) VALUES %s - ON DUPLICATE KEY UPDATE host_id = host_id`, strings.Join(parts, ",")), args...) + `+dialect.OnDuplicateKey("host_id,label_id", `host_id = VALUES(host_id)`), strings.Join(parts, ",")), args...) if err != nil { return ctxerr.Wrap(ctx, err, "upsert label membership") } @@ -2171,6 +2194,12 @@ func (ds *Datastore) RestoreMDMApplePendingDEPHost(ctx context.Context, host *fl // limited subset of fields just as if the host were initially ingested from DEP sync; // however, we also restore the UUID. Note that we are explicitly not restoring the // osquery_host_id. + // PG uses GENERATED ALWAYS AS IDENTITY for the id column, so we need + // OVERRIDING SYSTEM VALUE to insert an explicit id. + overriding := "" + if ds.dialect.ReturningID() != "" { + overriding = " OVERRIDING SYSTEM VALUE" + } stmt := ` INSERT INTO hosts ( id, @@ -2183,7 +2212,7 @@ INSERT INTO hosts ( osquery_host_id, refetch_requested, team_id -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` +)` + overriding + ` VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` // Handle zero time values by converting them to nil for SQL NULL var lastEnrolledAt, detailUpdatedAt interface{} @@ -2214,14 +2243,14 @@ INSERT INTO hosts ( // Upsert related host tables for the restored host just as if it were initially ingested // from DEP sync. Note we are not upserting host_dep_assignments in order to preserve the // existing timestamps. - if err := upsertHostDisplayNames(ctx, tx, *host); err != nil { + if err := upsertHostDisplayNames(ctx, tx, ds.dialect, *host); err != nil { // TODO: Why didn't this work as expected? return ctxerr.Wrap(ctx, err, "restore pending dep host display name") } - if err := upsertMDMAppleHostLabelMembershipDB(ctx, tx, ds.logger, *host); err != nil { + if err := upsertMDMAppleHostLabelMembershipDB(ctx, tx, ds.dialect, ds.logger, *host); err != nil { return ctxerr.Wrap(ctx, err, "restore pending dep host label membership") } - if err := upsertMDMAppleHostMDMInfoDB(ctx, tx, ac, true, false, host.ID); err != nil { + if err := upsertMDMAppleHostMDMInfoDB(ctx, tx, ds.dialect, ac, true, false, host.ID); err != nil { return ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert MDM info") } @@ -2354,21 +2383,21 @@ WHERE identifier NOT IN (?) ` - const insertNewOrEditedProfile = ` + insertNewOrEditedProfile := ` INSERT INTO mdm_apple_configuration_profiles ( profile_uuid, team_id, identifier, name, scope, mobileconfig, checksum, uploaded_at, secrets_updated_at ) VALUES -- see https://stackoverflow.com/a/51393124/1094941 - ( CONCAT('` + fleet.MDMAppleProfileUUIDPrefix + `', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, ?, ?, UNHEX(MD5(mobileconfig)), CURRENT_TIMESTAMP(6), ?) -ON DUPLICATE KEY UPDATE + ( CONCAT('` + fleet.MDMAppleProfileUUIDPrefix + `', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, ?, ?, UNHEX(MD5(?)), CURRENT_TIMESTAMP(6), ?) +` + ds.dialect.OnDuplicateKey("team_id,identifier", ` uploaded_at = IF(checksum = VALUES(checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP(6)), secrets_updated_at = VALUES(secrets_updated_at), checksum = VALUES(checksum), name = VALUES(name), mobileconfig = VALUES(mobileconfig) -` +`) // use a profile team id of 0 if no-team var profTeamID uint @@ -2459,7 +2488,7 @@ ON DUPLICATE KEY UPDATE // contents is the same as it was already). for _, p := range incomingProfs { if result, err = tx.ExecContext(ctx, insertNewOrEditedProfile, profTeamID, p.Identifier, p.Name, p.Scope, - p.Mobileconfig, p.SecretsUpdatedAt); err != nil { + p.Mobileconfig, p.Mobileconfig, p.SecretsUpdatedAt); err != nil { return false, ctxerr.Wrapf(ctx, err, "insert new/edited profile with identifier %q", p.Identifier) } didInsertOrUpdate := insertOnDuplicateDidInsertOrUpdate(result) @@ -2807,15 +2836,15 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB( scope ) VALUES %s - ON DUPLICATE KEY UPDATE + %s + `, strings.TrimSuffix(valuePart, ","), ds.dialect.OnDuplicateKey("host_uuid,profile_uuid", ` operation_type = VALUES(operation_type), status = VALUES(status), command_uuid = VALUES(command_uuid), checksum = VALUES(checksum), secrets_updated_at = VALUES(secrets_updated_at), detail = VALUES(detail), - scope = VALUES(scope) - `, strings.TrimSuffix(valuePart, ",")) + scope = VALUES(scope)`)) _, err := tx.ExecContext(ctx, baseStmt, args...) return ctxerr.Wrap(ctx, err, "bulk set pending profile status execute batch") @@ -3054,9 +3083,9 @@ func generateDesiredStateQuery(entityType string) string { ne.type IN ('Device', 'User Enrollment (Device)') AND ( %s ) GROUP BY - mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.${checksumColumn}, mae.secrets_updated_at, mae.scope + mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.${checksumColumn}, mae.secrets_updated_at, mae.scope, nd.authenticate_at HAVING - ${countEntityLabelsColumn} > 0 AND count_host_labels = ${countEntityLabelsColumn} + COUNT(*) > 0 AND COUNT(lm.label_id) = COUNT(*) UNION @@ -3106,10 +3135,13 @@ func generateDesiredStateQuery(entityType string) string { ne.type IN ('Device', 'User Enrollment (Device)') AND ( %s ) GROUP BY - mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.${checksumColumn}, mae.secrets_updated_at, mae.scope + mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.${checksumColumn}, mae.secrets_updated_at, mae.scope, nd.authenticate_at HAVING -- considers only the profiles with labels, without any broken label, with results reported after all labels were created and with the host not in any label - ${countEntityLabelsColumn} > 0 AND ${countEntityLabelsColumn} = count_non_broken_labels AND ${countEntityLabelsColumn} = count_host_updated_after_labels AND count_host_labels = 0 + COUNT(*) > 0 AND COUNT(*) = COUNT(mel.label_id) AND COUNT(*) = SUM( + CASE WHEN lbl.label_membership_type <> 1 AND lbl.created_at IS NOT NULL AND h.label_updated_at >= lbl.created_at THEN 1 + WHEN lbl.label_membership_type = 1 AND lbl.created_at IS NOT NULL THEN 1 + ELSE 0 END) AND COUNT(lm.label_id) = 0 UNION @@ -3148,9 +3180,9 @@ func generateDesiredStateQuery(entityType string) string { ne.type IN ('Device', 'User Enrollment (Device)') AND ( %s ) GROUP BY - mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.${checksumColumn}, mae.secrets_updated_at, mae.scope + mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.${checksumColumn}, mae.secrets_updated_at, mae.scope, nd.authenticate_at HAVING - ${countEntityLabelsColumn} > 0 AND count_host_labels >= 1 + COUNT(*) > 0 AND COUNT(lm.label_id) >= 1 `, func(s string) string { return dynamicNames[s] }) } @@ -3434,7 +3466,8 @@ func (ds *Datastore) BulkUpsertMDMAppleHostProfiles(ctx context.Context, payload scope ) VALUES %s - ON DUPLICATE KEY UPDATE + %s`, + strings.TrimSuffix(valuePart, ","), ds.dialect.OnDuplicateKey("host_uuid,profile_uuid", fmt.Sprintf(` status = VALUES(status), operation_type = VALUES(operation_type), detail = VALUES(detail), @@ -3446,8 +3479,7 @@ func (ds *Datastore) BulkUpsertMDMAppleHostProfiles(ctx context.Context, payload profile_name = VALUES(profile_name), command_uuid = VALUES(command_uuid), variables_updated_at = VALUES(variables_updated_at), - scope = VALUES(scope)`, - strings.TrimSuffix(valuePart, ","), fleet.MDMOperationTypeRemove, + scope = VALUES(scope)`, fleet.MDMOperationTypeRemove)), ) // We need to run with retry due to deadlocks. @@ -3570,36 +3602,36 @@ func sqlCaseMDMAppleStatus() string { verified = fmt.Sprintf("'%s'", string(fleet.MDMDeliveryVerified)) ) return ` - CASE WHEN (prof_failed - OR decl_failed - OR fv_failed - OR rl_failed) THEN + CASE WHEN ((prof_failed != 0) + OR (decl_failed != 0) + OR (fv_failed != 0) + OR (rl_failed != 0)) THEN ` + failed + ` - WHEN (prof_pending - OR decl_pending - OR rl_pending + WHEN ((prof_pending != 0) + OR (decl_pending != 0) + OR (rl_pending != 0) -- special case for filevault, it's pending if the profile is -- pending OR the profile is verified or verifying but we still -- don't have an encryption key. - OR(fv_pending - OR((fv_verifying - OR fv_verified) + OR((fv_pending != 0) + OR(((fv_verifying != 0) + OR (fv_verified != 0)) AND (hdek.base64_encrypted IS NULL OR (hdek.decryptable IS NOT NULL AND hdek.decryptable != 1))))) THEN ` + pending + ` - WHEN (prof_verifying - OR decl_verifying - OR rl_verifying + WHEN ((prof_verifying != 0) + OR (decl_verifying != 0) + OR (rl_verifying != 0) -- special case when fv profile is verifying, and we already have an encryption key, in any state, we treat as verifying - OR(fv_verifying + OR((fv_verifying != 0) AND hdek.base64_encrypted IS NOT NULL AND (hdek.decryptable IS NULL OR hdek.decryptable = 1)) -- special case when fv profile is verified, but we didn't verify the encryption key, we treat as verifying - OR(fv_verified + OR((fv_verified != 0) AND hdek.base64_encrypted IS NOT NULL AND hdek.decryptable IS NULL)) THEN ` + verifying + ` - WHEN (prof_verified - OR decl_verified - OR rl_verified - OR(fv_verified + WHEN ((prof_verified != 0) + OR (decl_verified != 0) + OR (rl_verified != 0) + OR((fv_verified != 0) AND hdek.base64_encrypted IS NOT NULL AND hdek.decryptable = 1)) THEN ` + verified + ` END @@ -3708,6 +3740,7 @@ func sqlJoinMDMAppleDeclarationsStatus() string { func (ds *Datastore) GetMDMAppleProfilesSummary(ctx context.Context, teamID *uint) (*fleet.MDMProfilesSummary, error) { stmt := ` +SELECT count, status FROM ( SELECT COUNT(id) AS count, %s AS status @@ -3720,7 +3753,8 @@ FROM WHERE platform IN('darwin', 'ios', 'ipados') AND %s GROUP BY - status HAVING status IS NOT NULL` + status +) sq WHERE status IS NOT NULL` teamFilter := "team_id IS NULL" if teamID != nil && *teamID > 0 { @@ -3771,9 +3805,9 @@ func (ds *Datastore) InsertMDMIdPAccount(ctx context.Context, account *fleet.MDM (uuid, username, fullname, email) VALUES (COALESCE(NULLIF(TRIM(?), ''), UUID()), ?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("email", ` username = VALUES(username), - fullname = VALUES(fullname)` + fullname = VALUES(fullname)`) _, err := ds.writer(ctx).ExecContext(ctx, stmt, account.UUID, account.Username, account.Fullname, account.Email) return ctxerr.Wrap(ctx, err, "creating new MDM IdP account") @@ -3994,21 +4028,21 @@ func (ds *Datastore) BulkUpsertMDMAppleConfigProfiles(ctx context.Context, paylo teamID = *cp.TeamID } - args = append(args, teamID, cp.Identifier, cp.Name, cp.Scope, cp.Mobileconfig, cp.SecretsUpdatedAt) + args = append(args, teamID, cp.Identifier, cp.Name, cp.Scope, cp.Mobileconfig, cp.Mobileconfig, cp.SecretsUpdatedAt) // see https://stackoverflow.com/a/51393124/1094941 - sb.WriteString("( CONCAT('a', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, ?, ?, UNHEX(MD5(mobileconfig)), CURRENT_TIMESTAMP(), ?),") + sb.WriteString("( CONCAT('a', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, ?, ?, UNHEX(MD5(?)), CURRENT_TIMESTAMP(), ?),") } stmt := fmt.Sprintf(` INSERT INTO mdm_apple_configuration_profiles (profile_uuid, team_id, identifier, name, scope, mobileconfig, checksum, uploaded_at, secrets_updated_at) VALUES %s - ON DUPLICATE KEY UPDATE + %s +`, strings.TrimSuffix(sb.String(), ","), ds.dialect.OnDuplicateKey("team_id,identifier", ` uploaded_at = IF(checksum = VALUES(checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP()), mobileconfig = VALUES(mobileconfig), checksum = VALUES(checksum), - secrets_updated_at = VALUES(secrets_updated_at) -`, strings.TrimSuffix(sb.String(), ",")) + secrets_updated_at = VALUES(secrets_updated_at)`)) if _, err := ds.writer(ctx).ExecContext(ctx, stmt, args...); err != nil { return ctxerr.Wrapf(ctx, err, "upsert mdm config profiles") @@ -4033,7 +4067,7 @@ func (ds *Datastore) InsertMDMAppleBootstrapPackage(ctx context.Context, bp *fle const insStmt = `INSERT INTO mdm_apple_bootstrap_packages (team_id, name, sha256, bytes, token) VALUES (?, ?, ?, ?, ?)` execInsert := func(args ...any) error { if _, err := ds.writer(ctx).ExecContext(ctx, insStmt, args...); err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return ctxerr.Wrap(ctx, alreadyExists("BootstrapPackage", fmt.Sprintf("for team %d", bp.TeamID))) } return ctxerr.Wrap(ctx, err, "create bootstrap package") @@ -4108,7 +4142,7 @@ WHERE team_id = 0 ` _, err := tx.ExecContext(ctx, insertStmt, toTeamID, uuid.New().String()) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return ctxerr.Wrap(ctx, &existsError{ ResourceType: "BootstrapPackage", TeamID: &toTeamID, @@ -4225,15 +4259,15 @@ func (ds *Datastore) GetMDMAppleBootstrapPackageSummary(ctx context.Context, tea } func (ds *Datastore) RecordSkippedHostBootstrapPackage(ctx context.Context, hostUUID string) error { - stmt := `INSERT INTO host_mdm_apple_bootstrap_packages (host_uuid, command_uuid, skipped) VALUES (?, NULL, 1) - ON DUPLICATE KEY UPDATE skipped = 1, command_uuid = NULL` + stmt := `INSERT INTO host_mdm_apple_bootstrap_packages (host_uuid, command_uuid, skipped) VALUES (?, NULL, TRUE) + ` + ds.dialect.OnDuplicateKey("host_uuid", `skipped = 1, command_uuid = NULL`) _, err := ds.writer(ctx).ExecContext(ctx, stmt, hostUUID) return ctxerr.Wrap(ctx, err, "record skipped bootstrap package") } func (ds *Datastore) RecordHostBootstrapPackage(ctx context.Context, commandUUID string, hostUUID string) error { - stmt := `INSERT INTO host_mdm_apple_bootstrap_packages (command_uuid, host_uuid, skipped) VALUES (?, ?, 0) - ON DUPLICATE KEY UPDATE command_uuid = command_uuid, skipped = 0` + stmt := `INSERT INTO host_mdm_apple_bootstrap_packages (command_uuid, host_uuid, skipped) VALUES (?, ?, FALSE) + ` + ds.dialect.OnDuplicateKey("host_uuid", `command_uuid = VALUES(command_uuid), skipped = 0`) _, err := ds.writer(ctx).ExecContext(ctx, stmt, commandUUID, hostUUID) return ctxerr.Wrap(ctx, err, "record bootstrap package command") } @@ -4366,22 +4400,25 @@ WHERE } func (ds *Datastore) SetOrUpdateMDMAppleSetupAssistant(ctx context.Context, asst *fleet.MDMAppleSetupAssistant) (*fleet.MDMAppleSetupAssistant, error) { - const stmt = ` + stmt := ` INSERT INTO mdm_apple_setup_assistants (team_id, global_or_team_id, name, profile) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("global_or_team_id", ` updated_at = IF(profile = VALUES(profile) AND name = VALUES(name), updated_at, CURRENT_TIMESTAMP), name = VALUES(name), profile = VALUES(profile) -` +`) var globalOrTmID uint if asst.TeamID != nil { globalOrTmID = *asst.TeamID } res, err := ds.writer(ctx).ExecContext(ctx, stmt, asst.TeamID, globalOrTmID, asst.Name, asst.Profile) if err != nil { + if isChildForeignKeyError(err) { + return nil, foreignKey("team", fmt.Sprintf("%d", globalOrTmID)) + } return nil, ctxerr.Wrap(ctx, err, "upsert mdm apple setup assistant") } @@ -4414,7 +4451,7 @@ func (ds *Datastore) SetMDMAppleSetupAssistantProfileUUID(ctx context.Context, t global_or_team_id = ? )` - const upsertStmt = ` + upsertStmt := ` INSERT INTO mdm_apple_setup_assistant_profiles ( setup_assistant_id, abm_token_id, profile_uuid ) ( @@ -4429,9 +4466,9 @@ func (ds *Datastore) SetMDMAppleSetupAssistantProfileUUID(ctx context.Context, t mas.id IS NOT NULL AND abt.id IS NOT NULL ) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("setup_assistant_id,abm_token_id", ` profile_uuid = VALUES(profile_uuid) - ` + `) var globalOrTmID uint if teamID != nil { @@ -4600,7 +4637,7 @@ func (ds *Datastore) SetMDMAppleDefaultSetupAssistantProfileUUID(ctx context.Con DELETE FROM mdm_apple_default_setup_assistants WHERE global_or_team_id = ?` - const upsertStmt = ` + upsertStmt := ` INSERT INTO mdm_apple_default_setup_assistants (team_id, global_or_team_id, profile_uuid, abm_token_id) SELECT @@ -4609,9 +4646,9 @@ func (ds *Datastore) SetMDMAppleDefaultSetupAssistantProfileUUID(ctx context.Con abm_tokens abt WHERE abt.organization_name = ? - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("id", ` profile_uuid = VALUES(profile_uuid) -` +`) var globalOrTmID uint if teamID != nil { globalOrTmID = *teamID @@ -4629,6 +4666,9 @@ func (ds *Datastore) SetMDMAppleDefaultSetupAssistantProfileUUID(ctx context.Con // upsert the profile uuid for the provided token _, err := ds.writer(ctx).ExecContext(ctx, upsertStmt, teamID, globalOrTmID, profileUUID, abmTokenOrgName) if err != nil { + if isChildForeignKeyError(err) { + return foreignKey("mdm_apple_default_setup_assistants", fmt.Sprintf("%d", globalOrTmID)) + } return ctxerr.Wrap(ctx, err, "upsert mdm apple default setup assistant") } return nil @@ -5177,7 +5217,7 @@ func (ds *Datastore) updateDeclarationsLabelAssociations(ctx context.Context, tx func (ds *Datastore) insertOrUpdateDeclarations(ctx context.Context, tx sqlx.ExtContext, incomingDeclarations []*fleet.MDMAppleDeclaration, teamID uint, ) (updatedDB bool, err error) { - const insertStmt = ` + insertStmt := ` INSERT INTO mdm_apple_declarations ( declaration_uuid, identifier, @@ -5190,13 +5230,13 @@ INSERT INTO mdm_apple_declarations ( VALUES ( ?,?,?,?,?,NOW(6),? ) -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("declaration_uuid", ` uploaded_at = IF(raw_json = VALUES(raw_json) AND name = VALUES(name) AND IFNULL(secrets_updated_at = VALUES(secrets_updated_at), TRUE), uploaded_at, NOW(6)), secrets_updated_at = VALUES(secrets_updated_at), name = VALUES(name), identifier = VALUES(identifier), raw_json = VALUES(raw_json) -` +`) for _, d := range incomingDeclarations { declUUID := fleet.MDMAppleDeclarationUUIDPrefix + uuid.NewString() @@ -5265,8 +5305,7 @@ func (ds *Datastore) getExistingDeclarations(ctx context.Context, tx sqlx.ExtCon const loadExistingDecls = ` SELECT name, - declaration_uuid, - raw_json + declaration_uuid FROM mdm_apple_declarations WHERE @@ -5319,7 +5358,7 @@ INSERT INTO mdm_apple_declarations ( } func (ds *Datastore) SetOrUpdateMDMAppleDeclaration(ctx context.Context, declaration *fleet.MDMAppleDeclaration) (*fleet.MDMAppleDeclaration, error) { - const stmt = ` + stmt := ` INSERT INTO mdm_apple_declarations ( declaration_uuid, team_id, @@ -5337,10 +5376,10 @@ INSERT INTO mdm_apple_declarations ( SELECT 1 FROM mdm_android_configuration_profiles WHERE name = ? AND team_id = ? ) ) -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("team_id, name", ` identifier = VALUES(identifier), uploaded_at = IF(raw_json = VALUES(raw_json) AND name = VALUES(name) AND IFNULL(secrets_updated_at = VALUES(secrets_updated_at), TRUE), uploaded_at, NOW(6)), - raw_json = VALUES(raw_json)` + raw_json = VALUES(raw_json)`) return ds.insertOrUpsertMDMAppleDeclaration(ctx, stmt, declaration) } @@ -5362,7 +5401,7 @@ func (ds *Datastore) insertOrUpsertMDMAppleDeclaration(ctx context.Context, insO declaration.Name, tmID, declaration.Name, tmID, declaration.Name, tmID) if err != nil { switch { - case IsDuplicate(err): + case ds.dialect.IsDuplicate(err): return ctxerr.Wrap(ctx, formatErrorDuplicateDeclaration(err, declaration)) default: return ctxerr.Wrap(ctx, err, "creating new apple mdm declaration") @@ -5597,7 +5636,7 @@ WHERE func (ds *Datastore) MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID string) ([]fleet.MDMAppleDDMDeclarationItem, error) { const stmt = ` SELECT - HEX(mad.token) as token, + COALESCE(HEX(mad.token), '') as token, mad.identifier, mad.declaration_uuid, status, operation_type, mad.uploaded_at FROM host_mdm_apple_declarations hmad @@ -5620,7 +5659,7 @@ func (ds *Datastore) MDMAppleDDMDeclarationsResponse(ctx context.Context, identi // declarations are removed, but the join would provide an extra layer of safety. const stmt = ` SELECT - mad.raw_json, HEX(mad.token) as token + mad.raw_json, COALESCE(HEX(mad.token), '') as token FROM host_mdm_apple_declarations hmad JOIN mdm_apple_declarations mad ON hmad.declaration_uuid = mad.declaration_uuid @@ -5642,7 +5681,7 @@ func (ds *Datastore) MDMAppleHostDeclarationsGetAndClearResync(ctx context.Conte stmt := ` SELECT DISTINCT host_uuid FROM host_mdm_apple_declarations - WHERE resync = '1' + WHERE resync = 1 ` err = sqlx.SelectContext(ctx, ds.reader(ctx), &hostUUIDs, stmt) if err != nil { @@ -5652,8 +5691,8 @@ func (ds *Datastore) MDMAppleHostDeclarationsGetAndClearResync(ctx context.Conte err = common_mysql.BatchProcessSimple(hostUUIDs, 1000, func(uuids []string) error { clearStmt := ` UPDATE host_mdm_apple_declarations - SET resync = '0' - WHERE host_uuid IN (?) AND resync = '1' + SET resync = 0 + WHERE host_uuid IN (?) AND resync = 1 ` clearStmt, args, err := sqlx.In(clearStmt, uuids) if err != nil { @@ -5924,7 +5963,7 @@ func mdmAppleGetHostsWithChangedDeclarationsDB(ctx context.Context, tx sqlx.ExtC SELECT ds.host_uuid, 'install' as operation_type, - ds.token, + COALESCE(ds.token, '') as token, ds.secrets_updated_at, ds.declaration_uuid, ds.declaration_identifier, @@ -5937,7 +5976,7 @@ func mdmAppleGetHostsWithChangedDeclarationsDB(ctx context.Context, tx sqlx.ExtC SELECT hmae.host_uuid, 'remove' as operation_type, - hmae.token, + COALESCE(hmae.token, '') as token, hmae.secrets_updated_at, hmae.declaration_uuid, hmae.declaration_identifier, @@ -5961,7 +6000,7 @@ func mdmAppleGetHostsWithChangedDeclarationsDB(ctx context.Context, tx sqlx.ExtC // MDMAppleStoreDDMStatusReport updates the status of the host's declarations. func (ds *Datastore) MDMAppleStoreDDMStatusReport(ctx context.Context, hostUUID string, updates []*fleet.MDMAppleHostDeclaration) error { getHostDeclarationsStmt := ` - SELECT host_uuid, status, operation_type, HEX(token) as token, secrets_updated_at, declaration_uuid, declaration_identifier, declaration_name + SELECT host_uuid, status, operation_type, COALESCE(HEX(token), '') as token, secrets_updated_at, declaration_uuid, declaration_identifier, declaration_name FROM host_mdm_apple_declarations WHERE host_uuid = ? ` @@ -5971,11 +6010,11 @@ INSERT INTO host_mdm_apple_declarations (host_uuid, declaration_uuid, status, operation_type, detail, declaration_name, declaration_identifier, token, secrets_updated_at) VALUES %s -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("host_uuid,declaration_uuid", ` status = VALUES(status), operation_type = VALUES(operation_type), detail = VALUES(detail) - ` + `) deletePendingRemovesStmt := ` DELETE FROM host_mdm_apple_declarations @@ -6325,12 +6364,12 @@ func (ds *Datastore) ReplaceMDMConfigAssets(ctx context.Context, assets []fleet. func (ds *Datastore) ListIOSAndIPadOSToRefetch(ctx context.Context, interval time.Duration) (devices []fleet.AppleDevicesToRefetch, err error, ) { - hostsStmt := ` -SELECT - h.id as host_id, - h.uuid as uuid, + hostsStmt := fmt.Sprintf(` +SELECT + h.id as host_id, + h.uuid as uuid, hmdm.installed_from_dep, - JSON_ARRAYAGG(hmc.command_type) as commands_already_sent + %s as commands_already_sent`, ds.dialect.JSONAgg("hmc.command_type")) + ` FROM hosts h INNER JOIN host_mdm hmdm ON hmdm.host_id = h.id INNER JOIN nano_enrollments ne ON ne.id = h.uuid @@ -6340,7 +6379,7 @@ WHERE AND TRIM(h.uuid) != '' AND TIMESTAMPDIFF(SECOND, h.detail_updated_at, NOW()) > ? AND ne.enabled = 1 -GROUP BY h.id` +GROUP BY h.id, h.uuid, hmdm.installed_from_dep` args := []any{fleet.ListAppleRefetchCommandPrefixes(), interval.Seconds()} hostsStmt, args, err = sqlx.In(hostsStmt, args...) if err != nil { @@ -6356,17 +6395,19 @@ GROUP BY h.id` func (ds *Datastore) GetEnrollmentIDsWithPendingMDMAppleCommands(ctx context.Context) (uuids []string, err error) { const stmt = ` -SELECT DISTINCT - neq.id -FROM - nano_enrollment_queue neq - LEFT JOIN nano_command_results ncr ON ncr.command_uuid = neq.command_uuid - AND ncr.id = neq.id -WHERE - neq.active = 1 - AND ncr.status IS NULL - AND neq.created_at >= NOW() - INTERVAL 7 DAY - AND neq.priority IN (0, 1) +SELECT id FROM ( + SELECT DISTINCT + neq.id + FROM + nano_enrollment_queue neq + LEFT JOIN nano_command_results ncr ON ncr.command_uuid = neq.command_uuid + AND ncr.id = neq.id + WHERE + neq.active = 1 + AND ncr.status IS NULL + AND neq.created_at >= NOW() - INTERVAL 7 DAY + AND neq.priority IN (0, 1) +) sub ORDER BY RAND() LIMIT 500 ` @@ -6441,9 +6482,7 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?) return nil, ctxerr.Wrap(ctx, err, "encrypt abm_token with datastore.serverPrivateKey") } - res, err := ds.writer(ctx).ExecContext( - ctx, - stmt, + tokenID, err := ds.insertAndGetID(ctx, ds.writer(ctx), stmt, tok.OrganizationName, tok.AppleID, tok.TermsExpired, @@ -6457,8 +6496,6 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?) return nil, ctxerr.Wrap(ctx, err, "inserting abm_token") } - tokenID, _ := res.LastInsertId() - tok.ID = uint(tokenID) //nolint:gosec // dismiss G115 cfg, err := ds.AppConfig(ctx) @@ -6749,11 +6786,11 @@ WHERE } func (ds *Datastore) AddHostMDMCommands(ctx context.Context, commands []fleet.HostMDMCommand) error { - const baseStmt = ` + baseStmt := ` INSERT INTO host_mdm_commands (host_id, command_type) VALUES %s - ON DUPLICATE KEY UPDATE - command_type = VALUES(command_type)` + ` + ds.dialect.OnDuplicateKey("host_id,command_type", ` + command_type = VALUES(command_type)`) for i := 0; i < len(commands); i += addHostMDMCommandsBatchSize { start := i @@ -6804,9 +6841,9 @@ func (ds *Datastore) CleanupHostMDMCommands(ctx context.Context) error { // Delete commands that don't have a corresponding host or have been sent over 1 day ago. // We are using 1 day instead of 7 days in case MDM commands fail to be sent or fail to process. They can be resent the next day. const stmt = ` - DELETE hmc FROM host_mdm_commands AS hmc - LEFT JOIN hosts h ON h.id = hmc.host_id - WHERE h.id IS NULL OR hmc.updated_at < NOW() - INTERVAL 1 DAY` + DELETE FROM host_mdm_commands + WHERE NOT EXISTS (SELECT 1 FROM hosts h WHERE h.id = host_mdm_commands.host_id) + OR host_mdm_commands.updated_at < NOW() - INTERVAL 1 DAY` if _, err := ds.writer(ctx).ExecContext(ctx, stmt); err != nil { return ctxerr.Wrap(ctx, err, "delete from host_mdm_commands") } @@ -6819,21 +6856,21 @@ func (ds *Datastore) CleanupHostMDMAppleProfiles(ctx context.Context) error { // This could also occur due to errors (i.e., large server/DB load) or server being stopped while processing the profiles. // After the entry is deleted, the mdm_apple_profile_manager job will try to requeue the profile. stmt := fmt.Sprintf(` - DELETE hmap FROM host_mdm_apple_profiles AS hmap + DELETE FROM host_mdm_apple_profiles WHERE ( - hmap.status IS NULL - OR hmap.status = '%s' + host_mdm_apple_profiles.status IS NULL + OR host_mdm_apple_profiles.status = '%s' ) - AND hmap.updated_at < NOW() - INTERVAL 1 HOUR + AND host_mdm_apple_profiles.updated_at < NOW() - INTERVAL 1 HOUR AND NOT EXISTS ( SELECT 1 FROM nano_enrollments ne - STRAIGHT_JOIN nano_enrollment_queue neq ON neq.id = ne.id - AND neq.command_uuid = hmap.command_uuid + JOIN nano_enrollment_queue neq ON neq.id = ne.id + AND neq.command_uuid = host_mdm_apple_profiles.command_uuid AND neq.active = 1 WHERE - ne.device_id = hmap.host_uuid + ne.device_id = host_mdm_apple_profiles.host_uuid AND ne.enabled = 1 );`, fleet.MDMDeliveryPending) @@ -6910,11 +6947,9 @@ func (ds *Datastore) ClearMDMUpcomingActivitiesDB(ctx context.Context, tx sqlx.E // the upcoming activities. const deleteUpcomingMDMActivities = ` DELETE FROM upcoming_activities - USING upcoming_activities - JOIN hosts h ON upcoming_activities.host_id = h.id WHERE - h.uuid = ? AND - upcoming_activities.activity_type IN ('vpp_app_install', 'in_house_app_install') + host_id IN (SELECT id FROM hosts WHERE uuid = ?) AND + activity_type IN ('vpp_app_install', 'in_house_app_install') ` _, err := tx.ExecContext(ctx, deleteUpcomingMDMActivities, hostUUID) if err != nil { @@ -7129,7 +7164,7 @@ func (ds *Datastore) AssociateHostMDMIdPAccountDB(ctx context.Context, hostUUID } func associateHostMDMIdPAccountDB(ctx context.Context, tx sqlx.ExtContext, hostUUID string, acctUUID string) error { - const stmt = ` + stmt := ` INSERT INTO host_mdm_idp_accounts (host_uuid, account_uuid) VALUES (?, ?) ON DUPLICATE KEY UPDATE @@ -7246,14 +7281,14 @@ func (ds *Datastore) SetLockCommandForLostModeCheckin(ctx context.Context, hostI } func (ds *Datastore) InsertHostLocationData(ctx context.Context, locData fleet.HostLocationData) error { - const stmt = ` + stmt := ` INSERT INTO host_last_known_locations (host_id, latitude, longitude) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("host_id", ` latitude = VALUES(latitude), longitude = VALUES(longitude) - ` + `) _, err := ds.writer(ctx).ExecContext(ctx, stmt, locData.HostID, locData.Latitude, locData.Longitude) return ctxerr.Wrap(ctx, err, "insert host location data") } @@ -7304,13 +7339,14 @@ func (ds *Datastore) SetHostsRecoveryLockPasswords(ctx context.Context, password stmt := ` INSERT INTO host_recovery_key_passwords (host_uuid, encrypted_password, status, operation_type) VALUES %s - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("host_uuid", ` encrypted_password = VALUES(encrypted_password), status = VALUES(status), operation_type = VALUES(operation_type), error_message = NULL, - deleted = 0 - ` + deleted = FALSE, + updated_at = CURRENT_TIMESTAMP + `) placeholders := strings.TrimSuffix(strings.Repeat("(?, ?, ?, ?),", len(passwords)), ",") stmt = fmt.Sprintf(stmt, placeholders) @@ -7323,12 +7359,11 @@ func (ds *Datastore) SetHostsRecoveryLockPasswords(ctx context.Context, password } func (ds *Datastore) GetHostRecoveryLockPassword(ctx context.Context, hostUUID string) (*fleet.HostRecoveryLockPassword, error) { - const stmt = `SELECT encrypted_password, updated_at, auto_rotate_at FROM host_recovery_key_passwords WHERE host_uuid = ? AND deleted = 0` + const stmt = `SELECT encrypted_password, updated_at FROM host_recovery_key_passwords WHERE host_uuid = ? AND deleted = 0` var row struct { - EncryptedPassword []byte `db:"encrypted_password"` - UpdatedAt time.Time `db:"updated_at"` - AutoRotateAt *time.Time `db:"auto_rotate_at"` + EncryptedPassword []byte `db:"encrypted_password"` + UpdatedAt time.Time `db:"updated_at"` } if err := sqlx.GetContext(ctx, ds.reader(ctx), &row, stmt, hostUUID); err != nil { if errors.Is(err, sql.ErrNoRows) { @@ -7344,9 +7379,8 @@ func (ds *Datastore) GetHostRecoveryLockPassword(ctx context.Context, hostUUID s } return &fleet.HostRecoveryLockPassword{ - Password: string(decrypted), - UpdatedAt: row.UpdatedAt, - AutoRotateAt: row.AutoRotateAt, + Password: string(decrypted), + UpdatedAt: row.UpdatedAt, }, nil } @@ -7395,7 +7429,7 @@ func (ds *Datastore) GetHostsForRecoveryLockAction(ctx context.Context) ([]strin // - Have no recovery lock password record OR have a password with NULL status (command not yet enqueued) // Note: hosts with status pending, verified, or failed are NOT included // Note: hosts with operation_type='remove' are handled by RestoreRecoveryLockForReenabledHosts - const stmt = ` + stmt := fmt.Sprintf(` SELECT h.uuid FROM hosts h JOIN nano_enrollments ne ON ne.device_id = h.uuid @@ -7404,20 +7438,21 @@ func (ds *Datastore) GetHostsForRecoveryLockAction(ctx context.Context) ([]strin CROSS JOIN app_config_json ac LEFT JOIN host_recovery_key_passwords rkp ON rkp.host_uuid = h.uuid AND rkp.deleted = 0 WHERE h.platform = 'darwin' - AND h.cpu_type LIKE '%arm%' + AND h.cpu_type LIKE '%%arm%%' AND ne.enabled = 1 AND ne.type IN ('Device', 'User Enrollment (Device)') AND hm.enrolled = 1 AND ( -- Team hosts: check team config - (h.team_id IS NOT NULL AND JSON_EXTRACT(t.config, '$.mdm.enable_recovery_lock_password') = true) + (h.team_id IS NOT NULL AND %s = 'true') OR -- No-team hosts: check appconfig - (h.team_id IS NULL AND JSON_EXTRACT(ac.json_value, '$.mdm.enable_recovery_lock_password') = true) + (h.team_id IS NULL AND %s = 'true') ) AND (rkp.host_uuid IS NULL OR rkp.status IS NULL) LIMIT 500 - ` + `, ds.dialect.JSONUnquoteExtract("t.config", "$.mdm.enable_recovery_lock_password"), + ds.dialect.JSONUnquoteExtract("ac.json_value", "$.mdm.enable_recovery_lock_password")) var hostUUIDs []string if err := sqlx.SelectContext(ctx, ds.reader(ctx), &hostUUIDs, stmt); err != nil { @@ -7444,7 +7479,30 @@ func (ds *Datastore) RestoreRecoveryLockForReenabledHosts(ctx context.Context) ( // Records with status='failed' (e.g., password mismatch) are NOT restored because: // - They represent terminal errors that require admin intervention // - Restoring them would mask the real problem and clear diagnostic error_message - stmt := fmt.Sprintf(` + var stmt string + if _, ok := ds.dialect.(postgresDialect); ok { + stmt = fmt.Sprintf(` + UPDATE host_recovery_key_passwords rkp + SET operation_type = '%s', + status = '%s', + error_message = NULL + FROM hosts h + LEFT JOIN teams t ON t.id = h.team_id + CROSS JOIN app_config_json ac + WHERE h.uuid = rkp.host_uuid + AND rkp.deleted = 0 + AND rkp.operation_type = '%s' + AND (rkp.status = '%s' OR rkp.status IS NULL) + AND ( + (h.team_id IS NOT NULL AND %s = 'true') + OR + (h.team_id IS NULL AND %s = 'true') + ) + `, fleet.MDMOperationTypeInstall, fleet.MDMDeliveryVerified, fleet.MDMOperationTypeRemove, fleet.MDMDeliveryPending, + ds.dialect.JSONUnquoteExtract("t.config", "$.mdm.enable_recovery_lock_password"), + ds.dialect.JSONUnquoteExtract("ac.json_value", "$.mdm.enable_recovery_lock_password")) + } else { + stmt = fmt.Sprintf(` UPDATE host_recovery_key_passwords rkp JOIN hosts h ON h.uuid = rkp.host_uuid LEFT JOIN teams t ON t.id = h.team_id @@ -7456,11 +7514,14 @@ func (ds *Datastore) RestoreRecoveryLockForReenabledHosts(ctx context.Context) ( AND rkp.operation_type = '%s' AND (rkp.status = '%s' OR rkp.status IS NULL) AND ( - (h.team_id IS NOT NULL AND JSON_EXTRACT(t.config, '$.mdm.enable_recovery_lock_password') = true) + (h.team_id IS NOT NULL AND %s = true) OR - (h.team_id IS NULL AND JSON_EXTRACT(ac.json_value, '$.mdm.enable_recovery_lock_password') = true) + (h.team_id IS NULL AND %s = true) ) - `, fleet.MDMOperationTypeInstall, fleet.MDMDeliveryVerified, fleet.MDMOperationTypeRemove, fleet.MDMDeliveryPending) + `, fleet.MDMOperationTypeInstall, fleet.MDMDeliveryVerified, fleet.MDMOperationTypeRemove, fleet.MDMDeliveryPending, + ds.dialect.JSONExtract("t.config", "$.mdm.enable_recovery_lock_password"), + ds.dialect.JSONExtract("ac.json_value", "$.mdm.enable_recovery_lock_password")) + } result, err := ds.writer(ctx).ExecContext(ctx, stmt) if err != nil { @@ -7557,13 +7618,15 @@ func (ds *Datastore) ClaimHostsForRecoveryLockClear(ctx context.Context) ([]stri (rkp.operation_type = '%s' AND rkp.status IS NULL) ) AND ( - (h.team_id IS NOT NULL AND JSON_EXTRACT(t.config, '$.mdm.enable_recovery_lock_password') = false) + (h.team_id IS NOT NULL AND %s != 'true') OR - (h.team_id IS NULL AND JSON_EXTRACT(ac.json_value, '$.mdm.enable_recovery_lock_password') = false) + (h.team_id IS NULL AND %s != 'true') ) LIMIT 500 FOR UPDATE - `, fleet.MDMOperationTypeInstall, fleet.MDMDeliveryVerified, fleet.MDMOperationTypeRemove) + `, fleet.MDMOperationTypeInstall, fleet.MDMDeliveryVerified, fleet.MDMOperationTypeRemove, + ds.dialect.JSONUnquoteExtract("t.config", "$.mdm.enable_recovery_lock_password"), + ds.dialect.JSONUnquoteExtract("ac.json_value", "$.mdm.enable_recovery_lock_password")) // Update all claimed hosts to remove/pending updateStmt := fmt.Sprintf(` @@ -7682,10 +7745,10 @@ func (ds *Datastore) InitiateRecoveryLockRotation(ctx context.Context, hostUUID } if dest.HasPending { - return ctxerr.Wrap(ctx, fleet.ErrRecoveryLockRotationPending, fmt.Sprintf("host %s", hostUUID)) + return ctxerr.Errorf(ctx, "rotation already pending for host %s", hostUUID) } - return ctxerr.Wrap(ctx, fleet.ErrRecoveryLockNotEligible, fmt.Sprintf("host %s (status=%v, operation_type=%v)", hostUUID, dest.Status.String, dest.OperationType.String)) + return ctxerr.Errorf(ctx, "host %s not eligible for rotation (status=%v, operation_type=%v)", hostUUID, dest.Status.String, dest.OperationType.String) } return nil @@ -7693,15 +7756,13 @@ func (ds *Datastore) InitiateRecoveryLockRotation(ctx context.Context, hostUUID func (ds *Datastore) CompleteRecoveryLockRotation(ctx context.Context, hostUUID string) error { // Move pending password to active and clear pending columns. - // Also clear auto_rotate_at since rotation is now complete. stmt := fmt.Sprintf(` UPDATE host_recovery_key_passwords SET encrypted_password = pending_encrypted_password, pending_encrypted_password = NULL, pending_error_message = NULL, status = '%s', - error_message = NULL, - auto_rotate_at = NULL + error_message = NULL WHERE host_uuid = ? AND deleted = 0 AND pending_encrypted_password IS NOT NULL diff --git a/server/datastore/mysql/ca_config_assets.go b/server/datastore/mysql/ca_config_assets.go index e5d8abbac6b..5f00b99a04e 100644 --- a/server/datastore/mysql/ca_config_assets.go +++ b/server/datastore/mysql/ca_config_assets.go @@ -56,10 +56,9 @@ func (ds *Datastore) saveCAConfigAssets(ctx context.Context, tx sqlx.ExtContext, stmt := fmt.Sprintf(` INSERT INTO ca_config_assets (name, type, value) VALUES %s - ON DUPLICATE KEY UPDATE - value = VALUES(value), - type = VALUES(type) - `, strings.TrimSuffix(strings.Repeat("(?,?,?),", len(assets)), ",")) + `+ds.dialect.OnDuplicateKey("name", `value = VALUES(value), + type = VALUES(type)`), + strings.TrimSuffix(strings.Repeat("(?,?,?),", len(assets)), ",")) args := make([]interface{}, 0, len(assets)*3) for _, asset := range assets { diff --git a/server/datastore/mysql/calendar_events.go b/server/datastore/mysql/calendar_events.go index 455e246dde4..fbb05c5e55f 100644 --- a/server/datastore/mysql/calendar_events.go +++ b/server/datastore/mysql/calendar_events.go @@ -33,7 +33,7 @@ func (ds *Datastore) CreateOrUpdateCalendarEvent( } var id int64 if err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - const calendarEventsQuery = ` + calendarEventsQuery := ` INSERT INTO calendar_events ( uuid_bin, email, @@ -42,16 +42,16 @@ func (ds *Datastore) CreateOrUpdateCalendarEvent( event, timezone ) VALUES (?, ?, ?, ?, ?, ?) - ON DUPLICATE KEY UPDATE - uuid_bin = VALUES(uuid_bin), + ` + ds.dialect.OnDuplicateKey("email", `uuid_bin = VALUES(uuid_bin), start_time = VALUES(start_time), end_time = VALUES(end_time), event = VALUES(event), timezone = VALUES(timezone), - updated_at = CURRENT_TIMESTAMP; - ` - result, err := tx.ExecContext( + updated_at = CURRENT_TIMESTAMP`) + id, err = insertAndGetIDTx( ctx, + tx, + ds.dialect, calendarEventsQuery, UUID[:], email, @@ -63,26 +63,23 @@ func (ds *Datastore) CreateOrUpdateCalendarEvent( if err != nil { return ctxerr.Wrap(ctx, err, "insert calendar event") } - - if insertOnDuplicateDidInsertOrUpdate(result) { - id, _ = result.LastInsertId() - } else { + if id == 0 { + // ON DUPLICATE KEY UPDATE did not insert a new row (MySQL returns 0 for LastInsertId); + // fall back to querying the existing row's ID. stmt := `SELECT id FROM calendar_events WHERE email = ?` if err := sqlx.GetContext(ctx, tx, &id, stmt, email); err != nil { return ctxerr.Wrap(ctx, err, "calendar event id") } } - const hostCalendarEventsQuery = ` + hostCalendarEventsQuery := ` INSERT INTO host_calendar_events ( host_id, calendar_event_id, webhook_status ) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE - webhook_status = VALUES(webhook_status), - calendar_event_id = VALUES(calendar_event_id); - ` + ` + ds.dialect.OnDuplicateKey("host_id", `webhook_status = VALUES(webhook_status), + calendar_event_id = VALUES(calendar_event_id)`) _, err = tx.ExecContext( ctx, hostCalendarEventsQuery, diff --git a/server/datastore/mysql/campaigns.go b/server/datastore/mysql/campaigns.go index f3cb58052ae..fb9c9f75235 100644 --- a/server/datastore/mysql/campaigns.go +++ b/server/datastore/mysql/campaigns.go @@ -48,12 +48,11 @@ func (ds *Datastore) NewDistributedQueryCampaign(ctx context.Context, camp *flee ) VALUES(?,?,?%s) `, createdAtField, createdAtPlaceholder) - result, err := ds.writer(ctx).ExecContext(ctx, sqlStatement, args...) + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), sqlStatement, args...) if err != nil { return nil, ctxerr.Wrap(ctx, err, "inserting distributed query campaign") } - id, _ := result.LastInsertId() camp.ID = uint(id) //nolint:gosec // dismiss G115 return camp, nil } @@ -140,12 +139,11 @@ func (ds *Datastore) NewDistributedQueryCampaignTarget(ctx context.Context, targ ) VALUES (?,?,?) ` - result, err := ds.writer(ctx).ExecContext(ctx, sqlStatement, target.Type, target.DistributedQueryCampaignID, target.TargetID) + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), sqlStatement, target.Type, target.DistributedQueryCampaignID, target.TargetID) if err != nil { return nil, ctxerr.Wrap(ctx, err, "insert distributed campaign target") } - id, _ := result.LastInsertId() target.ID = uint(id) //nolint:gosec // dismiss G115 return target, nil } diff --git a/server/datastore/mysql/carves.go b/server/datastore/mysql/carves.go index 4fea620a177..0c250f86e28 100644 --- a/server/datastore/mysql/carves.go +++ b/server/datastore/mysql/carves.go @@ -11,7 +11,7 @@ import ( "github.com/jmoiron/sqlx" ) -func upsertCarveDB(ctx context.Context, writer sqlx.ExecerContext, metadata *fleet.CarveMetadata) (int64, error) { +func upsertCarveDB(ctx context.Context, writer sqlx.ExtContext, dialect DialectHelper, metadata *fleet.CarveMetadata) (int64, error) { stmt := `INSERT INTO carve_metadata ( host_id, created_at, @@ -36,8 +36,10 @@ func upsertCarveDB(ctx context.Context, writer sqlx.ExecerContext, metadata *fle ? )` - result, err := writer.ExecContext( + id, err := insertAndGetIDTx( ctx, + writer, + dialect, stmt, metadata.HostId, metadata.CreatedAt.Format(mySQLTimestampFormat), @@ -53,11 +55,11 @@ func upsertCarveDB(ctx context.Context, writer sqlx.ExecerContext, metadata *fle if err != nil { return 0, ctxerr.Wrap(ctx, err, "insert carve metadata") } - return result.LastInsertId() + return id, nil } func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata) (*fleet.CarveMetadata, error) { - id, err := upsertCarveDB(ctx, ds.writer(ctx), metadata) + id, err := upsertCarveDB(ctx, ds.writer(ctx), ds.dialect, metadata) if err != nil { return nil, ctxerr.Wrap(ctx, err, "insert carve metadata") } diff --git a/server/datastore/mysql/certificate_authorities.go b/server/datastore/mysql/certificate_authorities.go index ce208316c67..b9674cad9ec 100644 --- a/server/datastore/mysql/certificate_authorities.go +++ b/server/datastore/mysql/certificate_authorities.go @@ -195,17 +195,13 @@ func (ds *Datastore) NewCertificateAuthority(ctx context.Context, ca *fleet.Cert return nil, err } - result, err := ds.writer(ctx).ExecContext(ctx, fmt.Sprintf(sqlInsertCertificateAuthority, placeholders), args...) + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), fmt.Sprintf(sqlInsertCertificateAuthority, placeholders), args...) if err != nil { if strings.Contains(err.Error(), "idx_ca_type_name") { return nil, fleet.ConflictError{Message: "a certificate authority with this name already exists"} } return nil, ctxerr.Wrap(ctx, err, "inserting new certificate authority") } - id, err := result.LastInsertId() - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "getting last insert ID for new certificate authority") - } ca.ID = uint(id) //nolint:gosec // dismiss G115 return ca, nil } @@ -230,7 +226,8 @@ const sqlInsertCertificateAuthority = `INSERT INTO certificate_authorities ( client_secret_encrypted ) VALUES %s` -const sqlUpsertCertificateAuthority = sqlInsertCertificateAuthority + ` ON DUPLICATE KEY UPDATE +func sqlUpsertCertificateAuthority(dialect DialectHelper) string { + return sqlInsertCertificateAuthority + ` ` + dialect.OnDuplicateKey("name,type", ` type = VALUES(type), name = VALUES(name), url = VALUES(url), @@ -245,7 +242,8 @@ const sqlUpsertCertificateAuthority = sqlInsertCertificateAuthority + ` ON DUPLI password_encrypted = VALUES(password_encrypted), challenge_encrypted = VALUES(challenge_encrypted), client_id = VALUES(client_id), - client_secret_encrypted = VALUES(client_secret_encrypted)` + client_secret_encrypted = VALUES(client_secret_encrypted)`) +} func sqlGenerateArgsForInsertCertificateAuthority(ctx context.Context, serverPrivateKey string, ca *fleet.CertificateAuthority) ([]interface{}, string, error) { var upns []byte @@ -308,7 +306,7 @@ func sqlGenerateArgsForInsertCertificateAuthority(ctx context.Context, serverPri return args, placeholders, nil } -func batchUpsertCertificateAuthorities(ctx context.Context, tx sqlx.ExtContext, serverPrivateKey string, certificateAuthorities []*fleet.CertificateAuthority) error { +func batchUpsertCertificateAuthorities(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, serverPrivateKey string, certificateAuthorities []*fleet.CertificateAuthority) error { if len(certificateAuthorities) == 0 { return nil } @@ -325,7 +323,7 @@ func batchUpsertCertificateAuthorities(ctx context.Context, tx sqlx.ExtContext, placeholders.WriteString(fmt.Sprintf("%s,", p)) } - stmt := fmt.Sprintf(sqlUpsertCertificateAuthority, strings.TrimSuffix(placeholders.String(), ",")) + stmt := fmt.Sprintf(sqlUpsertCertificateAuthority(dialect), strings.TrimSuffix(placeholders.String(), ",")) if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { return ctxerr.Wrap(ctx, err, "upserting certificate authorities") @@ -334,7 +332,7 @@ func batchUpsertCertificateAuthorities(ctx context.Context, tx sqlx.ExtContext, return nil } -func batchDeleteCertificateAuthorities(ctx context.Context, tx sqlx.ExtContext, certificateAuthorities []*fleet.CertificateAuthority) error { +func (ds *Datastore) batchDeleteCertificateAuthorities(ctx context.Context, tx sqlx.ExtContext, certificateAuthorities []*fleet.CertificateAuthority) error { if len(certificateAuthorities) == 0 { return nil } @@ -350,7 +348,7 @@ func batchDeleteCertificateAuthorities(ctx context.Context, tx sqlx.ExtContext, _, err := tx.ExecContext(ctx, stmt, args...) if err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { return &fleet.ConflictError{ Message: "Couldn't delete certificate authority. " + fleet.DeleteCAReferencedByTemplatesErrMsg + ". Please remove the certificate templates first.", } @@ -368,10 +366,10 @@ func (ds *Datastore) BatchApplyCertificateAuthorities(ctx context.Context, ops f upserts = append(upserts, ops.Update...) return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - if err := batchUpsertCertificateAuthorities(ctx, tx, ds.serverPrivateKey, upserts); err != nil { + if err := batchUpsertCertificateAuthorities(ctx, tx, ds.dialect, ds.serverPrivateKey, upserts); err != nil { return err } - if err := batchDeleteCertificateAuthorities(ctx, tx, ops.Delete); err != nil { + if err := ds.batchDeleteCertificateAuthorities(ctx, tx, ops.Delete); err != nil { return err } return nil @@ -396,10 +394,21 @@ func (ds *Datastore) DeleteCertificateAuthority(ctx context.Context, certificate return nil, ctxerr.Wrapf(ctx, err, "check certificate authority existence") } + // PG test schema has no FK constraints, so check for referencing templates manually. + if ds.dialect.IsPostgres() { + var refCount int + if err := sqlx.GetContext(ctx, ds.reader(ctx), &refCount, + "SELECT COUNT(*) FROM certificate_templates WHERE certificate_authority_id = ?", certificateAuthorityID); err == nil && refCount > 0 { + return nil, fleet.ConflictError{ + Message: "Couldn't delete certificate authority. " + fleet.DeleteCAReferencedByTemplatesErrMsg + ". Please remove the certificate templates first.", + } + } + } + stmt = "DELETE FROM certificate_authorities WHERE id = ?" result, err := ds.writer(ctx).ExecContext(ctx, stmt, certificateAuthorityID) if err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { return nil, fleet.ConflictError{ Message: "Couldn't delete certificate authority. " + fleet.DeleteCAReferencedByTemplatesErrMsg + ". Please remove the certificate templates first.", } diff --git a/server/datastore/mysql/certificate_templates.go b/server/datastore/mysql/certificate_templates.go index 13e1ef5313d..7ea588bebe9 100644 --- a/server/datastore/mysql/certificate_templates.go +++ b/server/datastore/mysql/certificate_templates.go @@ -180,7 +180,7 @@ func (ds *Datastore) GetCertificateTemplatesByTeamID(ctx context.Context, teamID } func (ds *Datastore) CreateCertificateTemplate(ctx context.Context, certificateTemplate *fleet.CertificateTemplate) (*fleet.CertificateTemplateResponse, error) { - result, err := ds.writer(ctx).ExecContext(ctx, ` + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), ` INSERT INTO certificate_templates ( name, team_id, @@ -189,17 +189,12 @@ func (ds *Datastore) CreateCertificateTemplate(ctx context.Context, certificateT ) VALUES (?, ?, ?, ?) `, certificateTemplate.Name, certificateTemplate.TeamID, certificateTemplate.CertificateAuthorityID, certificateTemplate.SubjectName) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return nil, ctxerr.Wrap(ctx, alreadyExists("CertificateTemplate", certificateTemplate.Name), "inserting certificate_template") } return nil, ctxerr.Wrap(ctx, err, "inserting certificate_template") } - id, err := result.LastInsertId() - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "getting last insert id for certificate_template") - } - return &fleet.CertificateTemplateResponse{ CertificateTemplateResponseSummary: fleet.CertificateTemplateResponseSummary{ ID: uint(id), //nolint:gosec @@ -236,17 +231,24 @@ func (ds *Datastore) BatchUpsertCertificateTemplates(ctx context.Context, certif return nil, nil } - const sqlInsertCertificate = ` + var sqlInsertCertificate string + if ds.dialect.IsPostgres() { + // PG: ON CONFLICT DO NOTHING since the UPDATE only sets columns to themselves (no-op). + // This ensures RowsAffected()=0 for existing rows, so insertOnDuplicateDidInsertOrUpdate + // correctly detects no modification occurred. + sqlInsertCertificate = ds.dialect.InsertIgnoreInto() + ` certificate_templates ( + name, team_id, certificate_authority_id, subject_name + ) VALUES (?, ?, ?, ?)` + ds.dialect.OnConflictDoNothing("team_id,name") + } else { + sqlInsertCertificate = ` INSERT INTO certificate_templates ( - name, - team_id, - certificate_authority_id, - subject_name + name, team_id, certificate_authority_id, subject_name ) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("team_id,name", ` name = VALUES(name), team_id = VALUES(team_id) - ` + `) + } teamsModifiedSet := make(map[uint]struct{}) for _, cert := range certificateTemplates { @@ -306,8 +308,7 @@ SELECT name, status, detail, - operation_type, - certificate_template_id + operation_type FROM host_certificate_templates WHERE host_uuid = ?` @@ -351,7 +352,7 @@ func (ds *Datastore) CreatePendingCertificateTemplatesForExistingHosts( (hosts.team_id = ? OR (? = 0 AND hosts.team_id IS NULL)) AND hosts.platform = '%s' AND host_mdm.enrolled = 1 - ON DUPLICATE KEY UPDATE host_uuid = host_uuid + `+ds.dialect.OnDuplicateKey("host_uuid,certificate_template_id", `host_uuid = VALUES(host_uuid)`)+` `, fleet.CertificateTemplatePending, fleet.MDMOperationTypeInstall, fleet.AndroidPlatform) result, err := ds.writer(ctx).ExecContext(ctx, stmt, certificateTemplateID, teamID, teamID) if err != nil { @@ -386,12 +387,12 @@ func (ds *Datastore) CreatePendingCertificateTemplatesForNewHost( UUID_TO_BIN(UUID(), true) FROM certificate_templates WHERE team_id = ? - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("host_uuid,certificate_template_id", ` -- allow 'remove' to transition to 'pending install', generating new uuid - uuid = IF(operation_type = '%s', UUID_TO_BIN(UUID(), true), uuid), - status = IF(operation_type = '%s', '%s', status), - operation_type = IF(operation_type = '%s', '%s', operation_type) - `, fleet.CertificateTemplatePending, fleet.MDMOperationTypeInstall, + uuid = IF(host_certificate_templates.operation_type = '%s', UUID_TO_BIN(UUID(), true), host_certificate_templates.uuid), + status = IF(host_certificate_templates.operation_type = '%s', '%s', host_certificate_templates.status), + operation_type = IF(host_certificate_templates.operation_type = '%s', '%s', host_certificate_templates.operation_type) + `), fleet.CertificateTemplatePending, fleet.MDMOperationTypeInstall, fleet.MDMOperationTypeRemove, fleet.MDMOperationTypeRemove, fleet.CertificateTemplatePending, fleet.MDMOperationTypeRemove, fleet.MDMOperationTypeInstall) @@ -410,48 +411,20 @@ func (ds *Datastore) ResendHostCertificateTemplate(ctx context.Context, hostID u hosts h ON h.uuid = hct.host_uuid SET hct.uuid = UUID_TO_BIN(UUID(), true), - hct.fleet_challenge = NULL, - hct.not_valid_before = NULL, - hct.not_valid_after = NULL, - hct.serial = NULL, - hct.detail = NULL, hct.status = ? WHERE h.id = ? AND hct.certificate_template_id = ? - ` - - const deleteChallenge = ` - DELETE c FROM - challenges c - INNER JOIN - host_certificate_templates hct ON hct.fleet_challenge = c.challenge - INNER JOIN - hosts h ON h.uuid = hct.host_uuid - WHERE - h.id = ? AND - hct.certificate_template_id = ? - ` - - if err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { - _, err := tx.ExecContext(ctx, deleteChallenge, hostID, templateID) - if err != nil { - return ctxerr.Wrap(ctx, err, "deleting challenges associated with resent certificate template") - } - - results, err := tx.ExecContext(ctx, stmt, fleet.CertificateTemplatePending, hostID, templateID) - if err != nil { - return ctxerr.Wrap(ctx, err, "updating host certificate template uuid") - } + ` - affected, _ := results.RowsAffected() - if affected == 0 { - return ctxerr.Wrapf(ctx, notFound("HostCertificateTemplate"), "template %d does not exist for host %d", templateID, hostID) - } + results, err := ds.writer(ctx).ExecContext(ctx, stmt, fleet.CertificateTemplatePending, hostID, templateID) + if err != nil { + return ctxerr.Wrap(ctx, err, "updating host certificate template uuid") + } - return nil - }); err != nil { - return ctxerr.Wrap(ctx, err, "resetting host certificate template for resend") + affected, _ := results.RowsAffected() + if affected == 0 { + return ctxerr.Wrapf(ctx, notFound("HostCertificateTemplate"), "template %d does not exist for host %d", templateID, hostID) } return nil diff --git a/server/datastore/mysql/conditional_access_bypass.go b/server/datastore/mysql/conditional_access_bypass.go index 12deee4da6b..70cdb36c54a 100644 --- a/server/datastore/mysql/conditional_access_bypass.go +++ b/server/datastore/mysql/conditional_access_bypass.go @@ -23,15 +23,14 @@ func (ds *Datastore) ConditionalAccessBypassDevice(ctx context.Context, hostID u pm.host_id = ? AND p.conditional_access_enabled = 1 AND p.critical = 1 - AND pm.passes = 0 + AND pm.passes IS FALSE ` - const insertStmt = ` + insertStmt := ` INSERT INTO host_conditional_access (host_id, bypassed_at) VALUES - (?, NOW(6)) - ON DUPLICATE KEY UPDATE - bypassed_at = NOW(6)` + (?, NOW()) + ` + ds.dialect.OnDuplicateKey("host_id", `bypassed_at = NOW()`) var blockCount uint diff --git a/server/datastore/mysql/conditional_access_microsoft.go b/server/datastore/mysql/conditional_access_microsoft.go index a4367f62d95..5cb707f152b 100644 --- a/server/datastore/mysql/conditional_access_microsoft.go +++ b/server/datastore/mysql/conditional_access_microsoft.go @@ -141,11 +141,10 @@ func (ds *Datastore) CreateHostConditionalAccessStatus(ctx context.Context, host `INSERT INTO microsoft_compliance_partner_host_statuses (host_id, device_id, user_principal_name) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE - device_id = VALUES(device_id), + `+ds.dialect.OnDuplicateKey("host_id", `device_id = VALUES(device_id), user_principal_name = VALUES(user_principal_name), managed = NULL, - compliant = NULL`, + compliant = NULL`), hostID, deviceID, userPrincipalName, ); err != nil { return ctxerr.Wrap(ctx, err, "create host conditional access status") diff --git a/server/datastore/mysql/conditional_access_scep.go b/server/datastore/mysql/conditional_access_scep.go index 442aaefcfd7..f29e85eaa9d 100644 --- a/server/datastore/mysql/conditional_access_scep.go +++ b/server/datastore/mysql/conditional_access_scep.go @@ -61,7 +61,24 @@ func (ds *Datastore) RevokeOldConditionalAccessCerts(ctx context.Context, graceP // Explanation: // 1. Find the newest "stable" cert for each host (stable = issued before grace period) // 2. Revoke all certs with serial < newest stable serial for that host - stmt := ` + var stmt string + if ds.dialect.IsPostgres() { + stmt = ` + UPDATE conditional_access_scep_certificates AS old_certs + SET revoked = true, updated_at = NOW() + FROM ( + SELECT host_id, MAX(serial) as newest_stable_serial + FROM conditional_access_scep_certificates + WHERE not_valid_before < NOW() - make_interval(secs => ?) + AND revoked = false + GROUP BY host_id + ) stable_certs + WHERE old_certs.host_id = stable_certs.host_id + AND old_certs.serial < stable_certs.newest_stable_serial + AND old_certs.revoked = false + ` + } else { + stmt = ` UPDATE conditional_access_scep_certificates old_certs INNER JOIN ( SELECT host_id, MAX(serial) as newest_stable_serial @@ -73,7 +90,8 @@ func (ds *Datastore) RevokeOldConditionalAccessCerts(ctx context.Context, graceP SET old_certs.revoked = 1, old_certs.updated_at = NOW(6) WHERE old_certs.serial < stable_certs.newest_stable_serial AND old_certs.revoked = 0 - ` + ` + } result, err := ds.writer(ctx).ExecContext(ctx, stmt, int(gracePeriod.Seconds())) if err != nil { diff --git a/server/datastore/mysql/cron_stats.go b/server/datastore/mysql/cron_stats.go index a761a197275..bf84174182c 100644 --- a/server/datastore/mysql/cron_stats.go +++ b/server/datastore/mysql/cron_stats.go @@ -53,14 +53,10 @@ UNION func (ds *Datastore) InsertCronStats(ctx context.Context, statsType fleet.CronStatsType, name string, instance string, status fleet.CronStatsStatus) (int, error) { stmt := `INSERT INTO cron_stats (stats_type, name, instance, status) VALUES (?, ?, ?, ?)` - res, err := ds.writer(ctx).ExecContext(ctx, stmt, statsType, name, instance, status) + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), stmt, statsType, name, instance, status) if err != nil { return 0, ctxerr.Wrap(ctx, err, "insert cron stats") } - id, err := res.LastInsertId() - if err != nil { - return 0, ctxerr.Wrap(ctx, err, "insert cron stats last insert id") - } return int(id), nil } @@ -119,17 +115,17 @@ func (ds *Datastore) CleanupCronStats(ctx context.Context) error { // WithAltLockID (e.g., "leader", "worker") store locks under a different name, so // the NOT EXISTS check won't find their lock and they fall back to the 2-hour timeout. updateStmt := ` - UPDATE cron_stats cs - SET cs.status = ? - WHERE cs.status IN (?, ?) + UPDATE cron_stats + SET status = ? + WHERE status IN (?, ?) AND ( - (cs.created_at < DATE_SUB(NOW(), INTERVAL 2 HOUR) + (created_at < DATE_SUB(NOW(), INTERVAL 2 HOUR) AND NOT EXISTS ( SELECT 1 FROM locks l - WHERE l.name = cs.name + WHERE l.name = cron_stats.name AND l.expires_at >= CURRENT_TIMESTAMP )) - OR cs.created_at < DATE_SUB(NOW(), INTERVAL 12 HOUR) + OR created_at < DATE_SUB(NOW(), INTERVAL 12 HOUR) )` if _, err := tx.ExecContext(ctx, updateStmt, fleet.CronStatsStatusExpired, fleet.CronStatsStatusPending, fleet.CronStatsStatusQueued); err != nil { return ctxerr.Wrap(ctx, err, "updating expired cron stats") diff --git a/server/datastore/mysql/delete.go b/server/datastore/mysql/delete.go index 47285401d94..082c7d121f5 100644 --- a/server/datastore/mysql/delete.go +++ b/server/datastore/mysql/delete.go @@ -29,7 +29,7 @@ func (ds *Datastore) deleteEntityByName(ctx context.Context, dbTable entity, nam deleteStmt := fmt.Sprintf("DELETE FROM %s WHERE name = ?", dbTable.name) result, err := ds.writer(ctx).ExecContext(ctx, deleteStmt, name) if err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { return ctxerr.Wrap(ctx, foreignKey(dbTable.name, name)) } return ctxerr.Wrapf(ctx, err, "delete %s", dbTable) diff --git a/server/datastore/mysql/disk_encryption.go b/server/datastore/mysql/disk_encryption.go index bc7a87de04d..0b657c8d6aa 100644 --- a/server/datastore/mysql/disk_encryption.go +++ b/server/datastore/mysql/disk_encryption.go @@ -206,7 +206,7 @@ func (ds *Datastore) ClearPendingEscrow(ctx context.Context, hostID uint) error func (ds *Datastore) ReportEscrowError(ctx context.Context, hostID uint, errorMessage string) error { _, err := ds.writer(ctx).ExecContext(ctx, ` INSERT INTO host_disk_encryption_keys - (host_id, base64_encrypted, client_error) VALUES (?, '', ?) ON DUPLICATE KEY UPDATE client_error = VALUES(client_error) + (host_id, base64_encrypted, client_error) VALUES (?, '', ?) `+ds.dialect.OnDuplicateKey("host_id", `client_error = VALUES(client_error)`)+` `, hostID, errorMessage) return err } @@ -214,7 +214,7 @@ INSERT INTO host_disk_encryption_keys func (ds *Datastore) QueueEscrow(ctx context.Context, hostID uint) error { _, err := ds.writer(ctx).ExecContext(ctx, ` INSERT INTO host_disk_encryption_keys - (host_id, base64_encrypted, reset_requested) VALUES (?, '', TRUE) ON DUPLICATE KEY UPDATE reset_requested = TRUE + (host_id, base64_encrypted, reset_requested) VALUES (?, '', TRUE) `+ds.dialect.OnDuplicateKey("host_id", `reset_requested = TRUE`)+` `, hostID) return err } diff --git a/server/datastore/mysql/errors.go b/server/datastore/mysql/errors.go index 2ae39305fe4..8d01394c7fd 100644 --- a/server/datastore/mysql/errors.go +++ b/server/datastore/mysql/errors.go @@ -14,6 +14,7 @@ import ( "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" common_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql" + pg "github.com/fleetdm/fleet/v4/server/platform/postgres" "github.com/go-sql-driver/mysql" ) @@ -86,6 +87,10 @@ func IsDuplicate(err error) bool { return true } } + // Also check PostgreSQL unique violation (SQLSTATE 23505) + if pg.IsDuplicate(err) { + return true + } return false } @@ -118,6 +123,9 @@ func isMySQLForeignKey(err error) bool { return true } } + if pg.IsForeignKey(err) { + return true + } return false } diff --git a/server/datastore/mysql/host_certificate_templates.go b/server/datastore/mysql/host_certificate_templates.go index c08dbc90f66..0c707aa9a01 100644 --- a/server/datastore/mysql/host_certificate_templates.go +++ b/server/datastore/mysql/host_certificate_templates.go @@ -152,10 +152,7 @@ func (ds *Datastore) GetHostCertificateTemplateRecord(ctx context.Context, hostU detail, COALESCE(BIN_TO_UUID(uuid, true), '') AS uuid, created_at, - updated_at, - not_valid_before, - not_valid_after, - serial + updated_at FROM host_certificate_templates WHERE host_uuid = ? AND certificate_template_id = ? ` @@ -620,7 +617,7 @@ func (ds *Datastore) GetAndroidCertificateTemplatesForRenewal( (DATEDIFF(not_valid_after, not_valid_before) > 30 AND not_valid_after < DATE_ADD(?, INTERVAL 30 DAY)) OR (DATEDIFF(not_valid_after, not_valid_before) > 2 AND DATEDIFF(not_valid_after, not_valid_before) <= 30 - AND not_valid_after < DATE_ADD(?, INTERVAL DATEDIFF(not_valid_after, not_valid_before)/2 DAY)) + AND not_valid_after < DATE_ADD(?, INTERVAL DATEDIFF(not_valid_after, not_valid_before)/2.0 DAY)) ) ORDER BY not_valid_after ASC LIMIT ? diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 7ec1b6dc7d9..4df51496b79 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -26,7 +26,7 @@ import ( "github.com/jmoiron/sqlx" ) -const hostHasIdentityCertSQL = `EXISTS(SELECT 1 FROM host_identity_scep_certificates hisc WHERE hisc.host_id = h.id AND hisc.revoked = 0)` +const hostHasIdentityCertSQL = `EXISTS(SELECT 1 FROM host_identity_scep_certificates hisc WHERE hisc.host_id = h.id AND hisc.revoked = false)` // Since many hosts may have issues, we need to batch the inserts of host issues. // This is a variable, so it can be adjusted during unit testing. @@ -138,9 +138,7 @@ func (ds *Datastore) NewHost(ctx context.Context, host *fleet.Host) (*fleet.Host ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ` - result, err := tx.ExecContext( - ctx, - sqlStatement, + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, sqlStatement, host.OsqueryHostID, host.DetailUpdatedAt, host.LabelUpdatedAt, @@ -166,7 +164,6 @@ func (ds *Datastore) NewHost(ctx context.Context, host *fleet.Host) (*fleet.Host if err != nil { return ctxerr.Wrap(ctx, err, "new host") } - id, _ := result.LastInsertId() host.ID = uint(id) _, err = tx.ExecContext(ctx, @@ -207,10 +204,10 @@ func (ds *Datastore) SerialUpdateHost(ctx context.Context, host *fleet.Host) err } func (ds *Datastore) SaveHostPackStats(ctx context.Context, teamID *uint, hostID uint, stats []fleet.PackStats) error { - return saveHostPackStatsDB(ctx, ds.writer(ctx), teamID, hostID, stats) + return saveHostPackStatsDB(ctx, ds.writer(ctx), ds.dialect, teamID, hostID, stats) } -func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, teamID *uint, hostID uint, stats []fleet.PackStats) error { +func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, dialect DialectHelper, teamID *uint, hostID uint, stats []fleet.PackStats) error { // NOTE: this implementation must be kept in sync with the async/batch version // in AsyncBatchSaveHostsScheduledQueryStats (in scheduled_queries.go) - that is, // the behaviour per host must be the same. @@ -282,47 +279,96 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, teamID *uint, hostID return nil } - if scheduledQueriesQueryCount > 0 { - // This query will import stats for queries (new format). - values := strings.TrimSuffix(strings.Repeat("((SELECT q.id FROM queries q WHERE COALESCE(q.team_id, 0) = ? AND q.name = ?),?,?,?,?,?,?,?,?,?,?),", scheduledQueriesQueryCount), ",") - sql := fmt.Sprintf(` - INSERT IGNORE INTO scheduled_query_stats ( - scheduled_query_id, - host_id, - average_memory, - denylisted, - executions, - schedule_interval, - last_executed, - output_size, - system_time, - user_time, - wall_time - ) - VALUES %s ON DUPLICATE KEY UPDATE - scheduled_query_id = VALUES(scheduled_query_id), - host_id = VALUES(host_id), - average_memory = VALUES(average_memory), - denylisted = VALUES(denylisted), - executions = VALUES(executions), - schedule_interval = VALUES(schedule_interval), - last_executed = VALUES(last_executed), - output_size = VALUES(output_size), - system_time = VALUES(system_time), - user_time = VALUES(user_time), - wall_time = VALUES(wall_time) - `, values) - if _, err := db.ExecContext(ctx, sql, scheduledQueriesArgs...); err != nil { - return ctxerr.Wrap(ctx, err, "insert query schedule stats") + // Deduplicate scheduled queries stats by (team_id, query_name) — last entry wins. + // PG's ON CONFLICT can't update the same row twice in a single INSERT. + if scheduledQueriesQueryCount > 1 { + type sqKey struct { + teamID uint + name string + } + argsPerRow := 12 // 2 (subquery) + 10 (values) + seen := make(map[sqKey]int) + for i := 0; i < scheduledQueriesQueryCount; i++ { + base := i * argsPerRow + key := sqKey{ + teamID: scheduledQueriesArgs[base].(uint), + name: scheduledQueriesArgs[base+1].(string), + } + seen[key] = i + } + if len(seen) < scheduledQueriesQueryCount { + var dedupedArgs []interface{} + dedupedCount := 0 + for i := 0; i < scheduledQueriesQueryCount; i++ { + base := i * argsPerRow + key := sqKey{ + teamID: scheduledQueriesArgs[base].(uint), + name: scheduledQueriesArgs[base+1].(string), + } + if seen[key] == i { // keep only last occurrence + dedupedArgs = append(dedupedArgs, scheduledQueriesArgs[base:base+argsPerRow]...) + dedupedCount++ + } + } + scheduledQueriesArgs = dedupedArgs + scheduledQueriesQueryCount = dedupedCount } } - if userPacksQueryCount > 0 { - // This query will import stats for 2017 packs. - // NOTE(lucas): If more than one scheduled query reference the same query then only one of the stats will be written. - values := strings.TrimSuffix(strings.Repeat("((SELECT sq.query_id FROM scheduled_queries sq JOIN packs p ON (sq.pack_id = p.id) WHERE p.pack_type IS NULL AND p.name = ? AND sq.name = ?),?,?,?,?,?,?,?,?,?,?),", userPacksQueryCount), ",") - sql := fmt.Sprintf(` - INSERT IGNORE INTO scheduled_query_stats ( + // Deduplicate user packs stats by (pack_name, query_name) — last entry wins. + // PG's ON CONFLICT can't update the same row twice in a single INSERT. + if userPacksQueryCount > 1 { + type packStatKey struct { + pack, query string + } + argsPerRow := 12 // 2 (subquery) + 10 (values) + seen := make(map[packStatKey]int) + for i := 0; i < userPacksQueryCount; i++ { + base := i * argsPerRow + key := packStatKey{ + pack: userPacksArgs[base].(string), + query: userPacksArgs[base+1].(string), + } + seen[key] = i + } + if len(seen) < userPacksQueryCount { + var dedupedArgs []interface{} + dedupedCount := 0 + for i := 0; i < userPacksQueryCount; i++ { + base := i * argsPerRow + key := packStatKey{ + pack: userPacksArgs[base].(string), + query: userPacksArgs[base+1].(string), + } + if seen[key] == i { // keep only last occurrence + dedupedArgs = append(dedupedArgs, userPacksArgs[base:base+argsPerRow]...) + dedupedCount++ + } + } + userPacksArgs = dedupedArgs + userPacksQueryCount = dedupedCount + } + } + + if scheduledQueriesQueryCount > 0 { + // This query will import stats for queries (new format). + if dialect.IsPostgres() { + // Uses INSERT...SELECT form so that rows where the query doesn't exist + // are naturally excluded (the SELECT returns 0 rows instead of NULL, + // which avoids NOT NULL violations on PG). + argsPerRow := 12 // 2 (subquery: teamID, name) + 10 (values) + var selectParts []string + var reorderedArgs []interface{} + for i := 0; i < scheduledQueriesQueryCount; i++ { + base := i * argsPerRow + selectParts = append(selectParts, + "SELECT q.id, ?::bigint,?::bigint,?::boolean,?::bigint,?::bigint,?::timestamptz,?::bigint,?::bigint,?::bigint,?::bigint FROM queries q WHERE COALESCE(q.team_id, 0) = ?::bigint AND q.name = ?::text") + // Reorder: value args first (host_id..wall_time), then subquery args (teamID, name) + reorderedArgs = append(reorderedArgs, scheduledQueriesArgs[base+2:base+argsPerRow]...) + reorderedArgs = append(reorderedArgs, scheduledQueriesArgs[base:base+2]...) + } + selectSQL := strings.Join(selectParts, " UNION ALL ") + sql := `INSERT INTO scheduled_query_stats ( scheduled_query_id, host_id, average_memory, @@ -335,7 +381,38 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, teamID *uint, hostID user_time, wall_time ) - VALUES %s ON DUPLICATE KEY UPDATE + ` + selectSQL + ` ` + dialect.OnDuplicateKey("host_id,scheduled_query_id,query_type", ` + scheduled_query_id = EXCLUDED.scheduled_query_id, + host_id = EXCLUDED.host_id, + average_memory = EXCLUDED.average_memory, + denylisted = EXCLUDED.denylisted, + executions = EXCLUDED.executions, + schedule_interval = EXCLUDED.schedule_interval, + last_executed = EXCLUDED.last_executed, + output_size = EXCLUDED.output_size, + system_time = EXCLUDED.system_time, + user_time = EXCLUDED.user_time, + wall_time = EXCLUDED.wall_time`) + ` + ` + if _, err := db.ExecContext(ctx, sql, reorderedArgs...); err != nil { + return ctxerr.Wrap(ctx, err, "insert query schedule stats") + } + } else { + values := strings.TrimSuffix(strings.Repeat("((SELECT q.id FROM queries q WHERE COALESCE(q.team_id, 0) = ? AND q.name = ?),?,?,?,?,?,?,?,?,?,?),", scheduledQueriesQueryCount), ",") + sql := fmt.Sprintf(dialect.InsertIgnoreInto()+` scheduled_query_stats ( + scheduled_query_id, + host_id, + average_memory, + denylisted, + executions, + schedule_interval, + last_executed, + output_size, + system_time, + user_time, + wall_time + ) + VALUES %s `+dialect.OnDuplicateKey("host_id,scheduled_query_id,query_type", ` scheduled_query_id = VALUES(scheduled_query_id), host_id = VALUES(host_id), average_memory = VALUES(average_memory), @@ -346,10 +423,98 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, teamID *uint, hostID output_size = VALUES(output_size), system_time = VALUES(system_time), user_time = VALUES(user_time), - wall_time = VALUES(wall_time) - `, values) - if _, err := db.ExecContext(ctx, sql, userPacksArgs...); err != nil { - return ctxerr.Wrap(ctx, err, "insert pack stats") + wall_time = VALUES(wall_time)`)+` + `, values) + if _, err := db.ExecContext(ctx, sql, scheduledQueriesArgs...); err != nil { + return ctxerr.Wrap(ctx, err, "insert query schedule stats") + } + } + } + + if userPacksQueryCount > 0 { + // This query will import stats for 2017 packs. + // NOTE(lucas): If more than one scheduled query reference the same query then only one of the stats will be written. + if dialect.IsPostgres() { + // Use INSERT...SELECT form so that rows where the subquery returns + // no match are naturally excluded (avoids NOT NULL violations on PG). + // Wrap in a subquery with DISTINCT ON to prevent "cannot affect row + // a second time" when multiple scheduled queries reference the same query_id. + argsPerRow := 12 // 2 (subquery: packName, sqName) + 10 (values) + var selectParts []string + var reorderedArgs []interface{} + for i := 0; i < userPacksQueryCount; i++ { + base := i * argsPerRow + selectParts = append(selectParts, + "SELECT sq.query_id, ?::bigint,?::bigint,?::boolean,?::bigint,?::bigint,?::timestamptz,?::bigint,?::bigint,?::bigint,?::bigint FROM scheduled_queries sq JOIN packs p ON (sq.pack_id = p.id) WHERE p.pack_type IS NULL AND p.name = ?::text AND sq.name = ?::text") + // Reorder: value args first (host_id..wall_time), then subquery args (packName, sqName) + reorderedArgs = append(reorderedArgs, userPacksArgs[base+2:base+argsPerRow]...) + reorderedArgs = append(reorderedArgs, userPacksArgs[base:base+2]...) + } + innerSQL := strings.Join(selectParts, " UNION ALL ") + sql := `INSERT INTO scheduled_query_stats ( + scheduled_query_id, + host_id, + average_memory, + denylisted, + executions, + schedule_interval, + last_executed, + output_size, + system_time, + user_time, + wall_time + ) + SELECT DISTINCT ON (scheduled_query_id) + scheduled_query_id::bigint, host_id::bigint, average_memory::bigint, denylisted::boolean, + executions::bigint, schedule_interval::bigint, last_executed::timestamptz, output_size::bigint, + system_time::bigint, user_time::bigint, wall_time::bigint + FROM (` + innerSQL + `) AS src(scheduled_query_id, host_id, average_memory, denylisted, executions, schedule_interval, last_executed, output_size, system_time, user_time, wall_time) + ` + dialect.OnDuplicateKey("host_id,scheduled_query_id,query_type", ` + scheduled_query_id = EXCLUDED.scheduled_query_id, + host_id = EXCLUDED.host_id, + average_memory = EXCLUDED.average_memory, + denylisted = EXCLUDED.denylisted, + executions = EXCLUDED.executions, + schedule_interval = EXCLUDED.schedule_interval, + last_executed = EXCLUDED.last_executed, + output_size = EXCLUDED.output_size, + system_time = EXCLUDED.system_time, + user_time = EXCLUDED.user_time, + wall_time = EXCLUDED.wall_time`) + if _, err := db.ExecContext(ctx, sql, reorderedArgs...); err != nil { + return ctxerr.Wrap(ctx, err, "insert pack stats") + } + } else { + values := strings.TrimSuffix(strings.Repeat("((SELECT sq.query_id FROM scheduled_queries sq JOIN packs p ON (sq.pack_id = p.id) WHERE p.pack_type IS NULL AND p.name = ? AND sq.name = ?),?,?,?,?,?,?,?,?,?,?),", userPacksQueryCount), ",") + sql := fmt.Sprintf(dialect.InsertIgnoreInto()+` scheduled_query_stats ( + scheduled_query_id, + host_id, + average_memory, + denylisted, + executions, + schedule_interval, + last_executed, + output_size, + system_time, + user_time, + wall_time + ) + VALUES %s `+dialect.OnDuplicateKey("host_id,scheduled_query_id,query_type", ` + scheduled_query_id = VALUES(scheduled_query_id), + host_id = VALUES(host_id), + average_memory = VALUES(average_memory), + denylisted = VALUES(denylisted), + executions = VALUES(executions), + schedule_interval = VALUES(schedule_interval), + last_executed = VALUES(last_executed), + output_size = VALUES(output_size), + system_time = VALUES(system_time), + user_time = VALUES(user_time), + wall_time = VALUES(wall_time)`)+` + `, values) + if _, err := db.ExecContext(ctx, sql, userPacksArgs...); err != nil { + return ctxerr.Wrap(ctx, err, "insert pack stats") + } } } @@ -358,7 +523,7 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, teamID *uint, hostID // loadhostPacksStatsDB will load all the "2017 pack" stats for the given host. The scheduled // queries that haven't run yet are returned with zero values. -func loadHostPackStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, hostPlatform string) ([]fleet.PackStats, error) { +func loadHostPackStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, hostPlatform string, dialect DialectHelper) ([]fleet.PackStats, error) { packs, err := listPacksForHost(ctx, db, hid) if err != nil { return nil, ctxerr.Wrapf(ctx, err, "list packs for host: %d", hid) @@ -372,7 +537,7 @@ func loadHostPackStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, packIDs[i] = packs[i].ID packTypes[packs[i].ID] = packs[i].Type } - ds := dialect.From(goqu.I("scheduled_queries").As("sq")).Select( + ds := dialect.GoquDialect().From(goqu.I("scheduled_queries").As("sq")).Select( goqu.I("sq.name").As("scheduled_query_name"), goqu.I("sq.id").As("scheduled_query_id"), goqu.I("sq.query_name").As("query_name"), @@ -380,16 +545,16 @@ func loadHostPackStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, goqu.I("p.name").As("pack_name"), goqu.I("p.id").As("pack_id"), goqu.COALESCE(goqu.I("sqs.average_memory"), 0).As("average_memory"), - goqu.COALESCE(goqu.I("sqs.denylisted"), false).As("denylisted"), + goqu.COALESCE(goqu.I("sqs.denylisted"), goqu.L("FALSE")).As("denylisted"), goqu.COALESCE(goqu.I("sqs.executions"), 0).As("executions"), goqu.I("sq.interval").As("schedule_interval"), - goqu.COALESCE(goqu.I("sqs.last_executed"), goqu.L("timestamp(?)", common_mysql.DefaultNonZeroTime)).As("last_executed"), + goqu.COALESCE(goqu.I("sqs.last_executed"), goqu.L("TIMESTAMP(?)", common_mysql.DefaultNonZeroTime)).As("last_executed"), goqu.COALESCE(goqu.I("sqs.output_size"), 0).As("output_size"), goqu.COALESCE(goqu.I("sqs.system_time"), 0).As("system_time"), goqu.COALESCE(goqu.I("sqs.user_time"), 0).As("user_time"), goqu.COALESCE(goqu.I("sqs.wall_time"), 0).As("wall_time"), ).Join( - dialect.From("packs").As("p").Select( + dialect.GoquDialect().From("packs").As("p").Select( goqu.I("id"), goqu.I("name"), ).Where(goqu.I("id").In(packIDs)), @@ -422,7 +587,7 @@ func loadHostPackStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, goqu.I("sq.platform").IsNull(), // scheduled_queries.platform can be a comma-separated list of // platforms, e.g. "darwin,windows". - goqu.L("FIND_IN_SET(?, sq.platform)", fleet.PlatformFromHost(hostPlatform)).Neq(0), + goqu.L(dialect.FindInSet("?", "sq.platform"), fleet.PlatformFromHost(hostPlatform)).Neq(0), ), ) sql, args, err := ds.ToSQL() @@ -453,7 +618,7 @@ func loadHostPackStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, // The filter is split into two statements joined by a UNION ALL to take advantage of indexes. // Using an OR in the WHERE clause causes a full table scan which causes issues with a large // queries table due to the high volume of live queries (created by zero trust workflows) -func loadHostScheduledQueryStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, hostPlatform string, teamID *uint) ([]fleet.QueryStats, error) { +func loadHostScheduledQueryStatsDB(ctx context.Context, db sqlx.QueryerContext, hid uint, hostPlatform string, teamID *uint, dialect DialectHelper) ([]fleet.QueryStats, error) { var teamID_ uint if teamID != nil { teamID_ = *teamID @@ -469,14 +634,14 @@ func loadHostScheduledQueryStatsDB(ctx context.Context, db sqlx.QueryerContext, q.discard_data, q.automations_enabled, MAX(qr.last_fetched) as last_fetched, - COALESCE(sqs.average_memory, 0) AS average_memory, - COALESCE(sqs.denylisted, false) AS denylisted, - COALESCE(sqs.executions, 0) AS executions, - COALESCE(sqs.last_executed, TIMESTAMP(?)) AS last_executed, - COALESCE(sqs.output_size, 0) AS output_size, - COALESCE(sqs.system_time, 0) AS system_time, - COALESCE(sqs.user_time, 0) AS user_time, - COALESCE(sqs.wall_time, 0) AS wall_time + COALESCE(MAX(sqs.average_memory), 0) AS average_memory, + COALESCE(MAX(sqs.denylisted), false) AS denylisted, + COALESCE(MAX(sqs.executions), 0) AS executions, + COALESCE(MAX(sqs.last_executed), TIMESTAMP(?)) AS last_executed, + COALESCE(MAX(sqs.output_size), 0) AS output_size, + COALESCE(MAX(sqs.system_time), 0) AS system_time, + COALESCE(MAX(sqs.user_time), 0) AS user_time, + COALESCE(MAX(sqs.wall_time), 0) AS wall_time FROM queries q LEFT JOIN @@ -494,10 +659,10 @@ func loadHostScheduledQueryStatsDB(ctx context.Context, db sqlx.QueryerContext, LEFT JOIN query_results qr ON (q.id = qr.query_id AND qr.host_id = ?) ` - filter1 := ` + filter1 := fmt.Sprintf(` WHERE - (q.platform = '' OR q.platform IS NULL OR FIND_IN_SET(?, q.platform) != 0) - AND q.is_scheduled = 1 + (q.platform = '' OR q.platform IS NULL OR %s != 0)`, dialect.FindInSet("?", "q.platform")) + ` + AND q.is_scheduled = true AND (q.automations_enabled IS TRUE OR (q.discard_data IS FALSE AND q.logging_type = ?)) AND (q.team_id IS NULL OR q.team_id = ?) GROUP BY q.id @@ -711,7 +876,7 @@ func deleteHosts(ctx context.Context, tx sqlx.ExtContext, hostIDs []uint) error // no point trying the uuid-based tables if the host's uuid is missing if len(hostUUIDs) != 0 { for table, col := range additionalHostRefsByUUID { - stmt, args, err := sqlx.In(fmt.Sprintf("DELETE FROM `%s` WHERE `%s` IN (?)", table, col), hostUUIDs) + stmt, args, err := sqlx.In(fmt.Sprintf(`DELETE FROM "%s" WHERE "%s" IN (?)`, table, col), hostUUIDs) if err != nil { return ctxerr.Wrapf(ctx, err, "building delete statement for %s for hosts %v", table, hostUUIDs) } @@ -880,12 +1045,12 @@ LIMIT host.DiskEncryptionEnabled = nil } - packStats, err := loadHostPackStatsDB(ctx, ds.reader(ctx), host.ID, host.Platform) + packStats, err := loadHostPackStatsDB(ctx, ds.reader(ctx), host.ID, host.Platform, ds.dialect) if err != nil { return nil, err } host.PackStats = packStats - queriesStats, err := loadHostScheduledQueryStatsDB(ctx, ds.reader(ctx), host.ID, host.Platform, host.TeamID) + queriesStats, err := loadHostScheduledQueryStatsDB(ctx, ds.reader(ctx), host.ID, host.Platform, host.TeamID, ds.dialect) if err != nil { return nil, err } @@ -959,6 +1124,7 @@ func queryStatsToScheduledQueryStats(queriesStats []fleet.QueryStats, packName s const hostMDMSelect = `, JSON_OBJECT( 'enrollment_status', hmdm.enrollment_status, + 'dep_profile_error', CASE WHEN hdep.assign_profile_response IN ('` + string(fleet.DEPAssignProfileResponseFailed) + `', '` + string(fleet.DEPAssignProfileResponseThrottled) + `') THEN CAST(TRUE AS JSON) @@ -966,7 +1132,7 @@ const hostMDMSelect = `, END, 'server_url', CASE - WHEN hmdm.is_server = 1 THEN NULL + WHEN hmdm.is_server = true THEN NULL ELSE hmdm.server_url END, 'encryption_key_available', @@ -976,7 +1142,7 @@ const hostMDMSelect = `, * unmarshaller was having problems converting int values to * booleans. */ - WHEN hdek.decryptable IS NULL OR hdek.decryptable = 0 THEN CAST(FALSE AS JSON) + WHEN hdek.decryptable IS NULL OR hdek.decryptable = false THEN CAST(FALSE AS JSON) ELSE CAST(TRUE AS JSON) END, 'raw_decryptable', @@ -995,14 +1161,14 @@ const hostMDMSelect = `, FROM mdm_windows_enrollments mwe WHERE mwe.host_uuid = h.uuid AND mwe.device_state = '` + microsoft_mdm.MDMDeviceStateEnrolled + `' - AND hmdm.enrolled = 1 + AND hmdm.enrolled = true ) THEN CAST(TRUE AS JSON) ELSE CAST(FALSE AS JSON) END ) WHEN h.platform = 'android' THEN - CASE WHEN hmdm.enrolled = 1 THEN CAST(TRUE AS JSON) ELSE CAST(FALSE AS JSON) END + CASE WHEN hmdm.enrolled = true THEN CAST(TRUE AS JSON) ELSE CAST(FALSE AS JSON) END WHEN h.platform IN ('ios', 'ipados', 'darwin') THEN (` + // NOTE: if you change any of the conditions in this // query, please update the AreHostsConnectedToFleetMDM @@ -1010,9 +1176,9 @@ const hostMDMSelect = `, `SELECT CASE WHEN EXISTS ( SELECT ne.id FROM nano_enrollments ne WHERE ne.id = h.uuid - AND ne.enabled = 1 + AND ne.enabled = true AND ne.type IN ('Device', 'User Enrollment (Device)') - AND hmdm.enrolled = 1 + AND hmdm.enrolled = true ) THEN CAST(TRUE AS JSON) ELSE CAST(FALSE AS JSON) @@ -1278,11 +1444,11 @@ func (ds *Datastore) applyHostFilters( deviceMappingJoin := fmt.Sprintf(`LEFT JOIN ( SELECT host_id, - CONCAT('[', GROUP_CONCAT(JSON_OBJECT('email', email, 'source', %s)), ']') AS device_mapping + CONCAT('[', %s, ']') AS device_mapping FROM host_emails GROUP BY - host_id) dm ON dm.host_id = h.id`, deviceMappingTranslateSourceColumn("")) + host_id) dm ON dm.host_id = h.id`, ds.dialect.GroupConcat(fmt.Sprintf("JSON_OBJECT('email', email, 'source', %s)", deviceMappingTranslateSourceColumn("")), ",")) if !opt.DeviceMapping { deviceMappingJoin = "" } @@ -1390,7 +1556,7 @@ func (ds *Datastore) applyHostFilters( opt.MacOSSettingsDiskEncryptionFilter.IsValid() || opt.OSSettingsDiskEncryptionFilter.IsValid() { connectedToFleetJoin = ` - LEFT JOIN nano_enrollments ne ON ne.id = h.uuid AND ne.enabled = 1 AND ne.type IN ('Device', 'User Enrollment (Device)') + LEFT JOIN nano_enrollments ne ON ne.id = h.uuid AND ne.enabled = true AND ne.type IN ('Device', 'User Enrollment (Device)') LEFT JOIN mdm_windows_enrollments mwe ON mwe.host_uuid = h.uuid AND mwe.device_state = ? LEFT JOIN android_devices ad ON ad.host_id = h.id` whereParams = append(whereParams, microsoft_mdm.MDMDeviceStateEnrolled) @@ -1535,17 +1701,17 @@ func (*Datastore) getBatchExecutionFilters(whereParams []interface{}, opt fleet. batchScriptExecutionJoin += ` LEFT JOIN host_script_results hsr ON bsehr.host_execution_id = hsr.execution_id` switch opt.BatchScriptExecutionStatusFilter { case fleet.BatchScriptExecutionRan: - batchScriptExecutionFilter += ` AND hsr.exit_code = 0 AND hsr.canceled = 0` + batchScriptExecutionFilter += ` AND hsr.exit_code = 0 AND hsr.canceled = false` case fleet.BatchScriptExecutionPending: // Pending can mean "waiting for execution" or "waiting for results". // hsr.exit_code IS NULL <- this means the script has not reported back - // (hsr.canceled IS NULL OR hsr.canceled = 0) <- this can mean the script is running, or that it hasn't been activated yet, + // (hsr.canceled IS NULL OR hsr.canceled = false) <- this can mean the script is running, or that it hasn't been activated yet, // but either way we haven't canceled it. // bsehr.error IS NULL <- this means the batch script framework didn't mark this host as incompatible // with this script run. - batchScriptExecutionFilter += ` AND ((hsr.host_id AND (hsr.exit_code IS NULL AND (hsr.canceled IS NULL OR hsr.canceled = 0) AND bsehr.error IS NULL)) OR (hsr.host_id is NULL AND ba.canceled = 0 AND bsehr.error IS NULL))` + batchScriptExecutionFilter += ` AND ((hsr.host_id IS NOT NULL AND (hsr.exit_code IS NULL AND (hsr.canceled IS NULL OR hsr.canceled = false) AND bsehr.error IS NULL)) OR (hsr.host_id is NULL AND ba.canceled = 0 AND bsehr.error IS NULL))` case fleet.BatchScriptExecutionErrored: - batchScriptExecutionFilter += ` AND hsr.exit_code <> 0 AND hsr.canceled = 0` + batchScriptExecutionFilter += ` AND hsr.exit_code <> 0 AND hsr.canceled = false` case fleet.BatchScriptExecutionIncompatible: batchScriptExecutionFilter += ` AND bsehr.error IS NOT NULL` case fleet.BatchScriptExecutionCanceled: @@ -1586,20 +1752,20 @@ func filterHostsByMDM(sql string, opt fleet.HostListOptions, params []interface{ params = append(params, *opt.MDMNameFilter) } if opt.MDMEnrollmentStatusFilter != "" { - // NOTE: ds.UpdateHostTablesOnMDMUnenroll sets installed_from_dep = 0 so DEP hosts are not counted as pending after unenrollment + // NOTE: ds.UpdateHostTablesOnMDMUnenroll sets installed_from_dep = false so DEP hosts are not counted as pending after unenrollment switch opt.MDMEnrollmentStatusFilter { case fleet.MDMEnrollStatusAutomatic: - sql += ` AND hmdm.enrolled = 1 AND hmdm.installed_from_dep = 1` + sql += ` AND hmdm.enrolled = true AND hmdm.installed_from_dep = true` case fleet.MDMEnrollStatusManual: - sql += ` AND hmdm.enrolled = 1 AND hmdm.installed_from_dep = 0 AND hmdm.is_personal_enrollment = 0` + sql += ` AND hmdm.enrolled = true AND hmdm.installed_from_dep = false AND hmdm.is_personal_enrollment = false` case fleet.MDMEnrollStatusPersonal: - sql += ` AND hmdm.enrolled = 1 AND hmdm.installed_from_dep = 0 AND hmdm.is_personal_enrollment = 1` + sql += ` AND hmdm.enrolled = true AND hmdm.installed_from_dep = false AND hmdm.is_personal_enrollment = true` case fleet.MDMEnrollStatusEnrolled: - sql += ` AND hmdm.enrolled = 1` + sql += ` AND hmdm.enrolled = true` case fleet.MDMEnrollStatusPending: - sql += ` AND hmdm.enrolled = 0 AND hmdm.installed_from_dep = 1` + sql += ` AND hmdm.enrolled = false AND hmdm.installed_from_dep = true` case fleet.MDMEnrollStatusUnenrolled: - sql += ` AND hmdm.enrolled = 0 AND hmdm.installed_from_dep = 0` + sql += ` AND hmdm.enrolled = false AND hmdm.installed_from_dep = false` } } if opt.MDMNameFilter != nil || opt.MDMIDFilter != nil || opt.MDMEnrollmentStatusFilter != "" { @@ -1664,7 +1830,7 @@ func filterHostsByMacOSSettingsStatus(sql string, opt fleet.HostListOptions, par } // ensure the host has MDM turned on - whereStatus := " AND ne.id IS NOT NULL AND hmdm.enrolled = 1" + whereStatus := " AND ne.id IS NOT NULL AND hmdm.enrolled = true" // macOS settings filter is not compatible with the "all teams" option so append the "no // team" filter here (note that filterHostsByTeam applies the "no team" filter if TeamFilter == 0) if opt.TeamFilter == nil { @@ -1698,7 +1864,7 @@ func filterHostsByMacOSDiskEncryptionStatus(sql string, opt fleet.HostListOption subquery, subqueryParams = subqueryFileVaultRemovingEnforcement() } - return sql + fmt.Sprintf(` AND EXISTS (%s) AND ne.id IS NOT NULL AND hmdm.enrolled = 1`, subquery), append(params, subqueryParams...) + return sql + fmt.Sprintf(` AND EXISTS (%s) AND ne.id IS NOT NULL AND hmdm.enrolled = true`, subquery), append(params, subqueryParams...) } func (ds *Datastore) filterHostsByOSSettingsStatus(ctx context.Context, sql string, opt fleet.HostListOptions, params []any, diskEncryptionConfig fleet.DiskEncryptionConfig) (string, []any, error) { @@ -1722,9 +1888,9 @@ func (ds *Datastore) filterHostsByOSSettingsStatus(ctx context.Context, sql stri } sqlFmt := ` AND ( - (h.platform = 'windows' AND mwe.host_uuid IS NOT NULL AND hmdm.enrolled = 1) -- windows - OR (h.platform IN ('darwin', 'ios', 'ipados') AND ne.id IS NOT NULL AND hmdm.enrolled = 1) -- apple - OR (h.platform = 'android' AND hmdm.enrolled = 1 AND ad.host_id IS NOT NULL) -- android + (h.platform = 'windows' AND mwe.host_uuid IS NOT NULL AND hmdm.enrolled = true) -- windows + OR (h.platform IN ('darwin', 'ios', 'ipados') AND ne.id IS NOT NULL AND hmdm.enrolled = true) -- apple + OR (h.platform = 'android' AND hmdm.enrolled = true AND ad.host_id IS NOT NULL) -- android OR ` + includeLinuxCond + ` )` @@ -1755,7 +1921,7 @@ AND ( paramsAndroid := []any{opt.OSSettingsFilter} // construct the WHERE for windows - whereWindows = `hmdm.is_server = 0` + whereWindows = `hmdm.is_server = false` paramsWindows := []any{} subqueryFailed, paramsFailed, err := subqueryHostsMDMWindowsOSSettingsStatusFailed() if err != nil { @@ -1881,8 +2047,8 @@ func (ds *Datastore) filterHostsByOSSettingsDiskEncryptionStatus(ctx context.Con sqlFmt += ` AND h.team_id IS NULL` } sqlFmt += ` AND ( - (h.platform = 'windows' AND mwe.host_uuid IS NOT NULL AND hmdm.enrolled = 1 AND hmdm.is_server = 0 AND %s) -- windows - OR (h.platform = 'darwin' AND ne.id IS NOT NULL AND hmdm.enrolled = 1 AND %s) -- apple + (h.platform = 'windows' AND mwe.host_uuid IS NOT NULL AND hmdm.enrolled = true AND hmdm.is_server = false AND %s) -- windows + OR (h.platform = 'darwin' AND ne.id IS NOT NULL AND hmdm.enrolled = true AND %s) -- apple OR ((h.platform = 'ubuntu' OR h.os_version LIKE 'Fedora%%') AND %s) -- linux )` @@ -1959,7 +2125,7 @@ func filterHostsByMDMBootstrapPackageStatus(sql string, opt fleet.HostListOption LEFT JOIN host_dep_assignments hda ON hda.host_id = hh.id WHERE - hh.id = h.id AND hmdm.installed_from_dep = 1` + hh.id = h.id AND hmdm.installed_from_dep = true` // NOTE: The approach below assumes that there is only one bootstrap package per host. If this // is not the case, then the query will need to be updated to use a GROUP BY and HAVING @@ -1969,7 +2135,7 @@ func filterHostsByMDMBootstrapPackageStatus(sql string, opt fleet.HostListOption subquery += ` AND ncr.status = 'Error'` case fleet.MDMBootstrapPackagePending: // Pending hosts exclude those that were skipped due to migration or will be skipped due to migration - subquery += ` AND (hmabp.skipped = 0 OR hmabp.skipped IS NULL) AND (hda.mdm_migration_deadline IS NULL OR (hda.mdm_migration_deadline = hda.mdm_migration_completed)) AND (ncr.status IS NULL OR (ncr.status != 'Acknowledged' AND ncr.status != 'Error'))` + subquery += ` AND (hmabp.skipped = false OR hmabp.skipped IS NULL) AND (hda.mdm_migration_deadline IS NULL OR (hda.mdm_migration_deadline = hda.mdm_migration_completed)) AND (ncr.status IS NULL OR (ncr.status != 'Acknowledged' AND ncr.status != 'Error'))` case fleet.MDMBootstrapPackageInstalled: subquery += ` AND ncr.status = 'Acknowledged'` } @@ -2483,9 +2649,9 @@ func (ds *Datastore) EnrollOrbit(ctx context.Context, opts ...fleet.DatastoreEnr hardware_model, platform, platform_like - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, true, ?, ?, ?, ?, ?, ?, ?) ` - result, err := tx.ExecContext(ctx, sqlInsert, + hostID, err := insertAndGetIDTx(ctx, tx, ds.dialect, sqlInsert, zeroTime, zeroTime, zeroTime, @@ -2505,7 +2671,6 @@ func (ds *Datastore) EnrollOrbit(ctx context.Context, opts ...fleet.DatastoreEnr if err != nil { return ctxerr.Wrap(ctx, err, "orbit enroll error inserting host details") } - hostID, _ := result.LastInsertId() const sqlHostDisplayName = ` INSERT INTO host_display_names (host_id, display_name) VALUES (?, ?) ` @@ -2585,14 +2750,13 @@ func (ds *Datastore) EnrollOsquery(ctx context.Context, opts ...fleet.DatastoreE refetch_requested, uuid, hardware_serial - ) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, true, ?, ?) ` - result, err := tx.ExecContext(ctx, sqlInsert, zeroTime, zeroTime, zeroTime, osqueryHostID, nodeKey, teamID, hardwareUUID, hardwareSerial) + lastInsertID, err := insertAndGetIDTx(ctx, tx, ds.dialect, sqlInsert, zeroTime, zeroTime, zeroTime, osqueryHostID, nodeKey, teamID, hardwareUUID, hardwareSerial) if err != nil { ds.logger.InfoContext(ctx, "host insert error", "err", err) return ctxerr.Wrap(ctx, err, "insert host") } - lastInsertID, _ := result.LastInsertId() const sqlHostDisplayName = ` INSERT INTO host_display_names (host_id, display_name) VALUES (?, '') ` @@ -2628,7 +2792,7 @@ func (ds *Datastore) EnrollOsquery(ctx context.Context, opts ...fleet.DatastoreE fmt.Sprintf("This is likely due to a duplicate UUID/identity identifier used by multiple hosts: %s", osqueryHostID)) } - if err := deleteAllPolicyMemberships(ctx, tx, enrolledHostInfo.ID); err != nil { + if err := deleteAllPolicyMemberships(ctx, tx, ds.dialect, enrolledHostInfo.ID); err != nil { return ctxerr.Wrap(ctx, err, "cleanup policy membership on re-enroll") } @@ -2677,7 +2841,7 @@ func (ds *Datastore) EnrollOsquery(ctx context.Context, opts ...fleet.DatastoreE _, err = tx.ExecContext(ctx, ` INSERT INTO host_seen_times (host_id, seen_time) VALUES (?, ?) - ON DUPLICATE KEY UPDATE seen_time = VALUES(seen_time)`, + `+ds.dialect.OnDuplicateKey("host_id", "seen_time = VALUES(seen_time)"), hostID, time.Now().UTC()) if err != nil { return ctxerr.Wrap(ctx, err, "new host seen time") @@ -2748,7 +2912,7 @@ func (ds *Datastore) EnrollOsquery(ctx context.Context, opts ...fleet.DatastoreE if err != nil { return ctxerr.Wrap(ctx, err, "getting the host to return") } - _, err = tx.ExecContext(ctx, `INSERT IGNORE INTO label_membership (host_id, label_id) VALUES (?, (SELECT id FROM labels WHERE name = 'All Hosts' AND label_type = 1))`, hostID) + _, err = tx.ExecContext(ctx, ds.dialect.InsertIgnoreInto()+` label_membership (host_id, label_id) VALUES (?, (SELECT id FROM labels WHERE name = 'All Hosts' AND label_type = 1))`+ds.dialect.OnConflictDoNothing("host_id,label_id"), hostID) if err != nil { return ctxerr.Wrap(ctx, err, "insert new host into all hosts label") } @@ -2769,7 +2933,7 @@ func (ds *Datastore) getContextTryStmt(ctx context.Context, dest interface{}, qu // nolint the statements are closed in Datastore.Close. if stmt := ds.loadOrPrepareStmt(ctx, query); stmt != nil { err := stmt.GetContext(ctx, dest, args...) - if err == nil || !isBadConnection(err) { + if err == nil || !ds.dialect.IsBadConnection(err) { return err } @@ -2907,7 +3071,7 @@ func (ds *Datastore) LoadHostByOrbitNodeKey(ctx context.Context, nodeKey string) h.policy_updated_at, h.public_ip, h.orbit_node_key, - IF(hdep.host_id AND ISNULL(hdep.deleted_at), true, false) AS dep_assigned_to_fleet, + (hdep.host_id IS NOT NULL AND hdep.deleted_at IS NULL) AS dep_assigned_to_fleet, hd.encrypted as disk_encryption_enabled, COALESCE(hdek.decryptable, false) as encryption_key_available, t.name as team_name, @@ -2998,7 +3162,7 @@ func (ds *Datastore) LoadHostByDeviceAuthToken(ctx context.Context, authToken st COALESCE(hd.percent_disk_space_available, 0) as percent_disk_space_available, COALESCE(hd.gigs_total_disk_space, 0) as gigs_total_disk_space, hd.encrypted as disk_encryption_enabled, - IF(hdep.host_id AND ISNULL(hdep.deleted_at), true, false) AS dep_assigned_to_fleet, + (hdep.host_id IS NOT NULL AND hdep.deleted_at IS NULL) AS dep_assigned_to_fleet, ` + hostHasIdentityCertSQL + ` as has_host_identity_cert FROM host_device_auth hda @@ -3039,19 +3203,18 @@ func (ds *Datastore) SetOrUpdateDeviceAuthToken(ctx context.Context, hostID uint // both the old and new tokens can be used for authentication during the transition // period (see #38351). If the current token is already expired (older than 1 hour, // matching deviceAuthTokenTTL), previous_token is set to NULL to avoid reviving it. - const stmt = ` + stmt := ` INSERT INTO host_device_auth ( host_id, token ) VALUES (?, ?) - ON DUPLICATE KEY UPDATE - previous_token = IF(token = VALUES(token), previous_token, - IF(updated_at >= DATE_SUB(NOW(), INTERVAL 3600 SECOND), token, NULL)), - token = VALUES(token) + ` + ds.dialect.OnDuplicateKey("host_id", `previous_token = IF(host_device_auth.token = VALUES(token), host_device_auth.previous_token, + IF(host_device_auth.updated_at >= DATE_SUB(NOW(), INTERVAL 3600 SECOND), host_device_auth.token, NULL)), + token = VALUES(token)`) + ` ` _, err := ds.writer(ctx).ExecContext(ctx, stmt, hostID, authToken) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return fleet.ConflictError{Message: "auth token conflicts with another host"} } return ctxerr.Wrap(ctx, err, "upsert host's device auth token") @@ -3093,7 +3256,7 @@ func (ds *Datastore) MarkHostsSeen(ctx context.Context, hostIDs []uint, t time.T insertValues := strings.TrimSuffix(strings.Repeat("(?, ?),", len(hostIDs)), ",") query := fmt.Sprintf(` INSERT INTO host_seen_times (host_id, seen_time) VALUES %s - ON DUPLICATE KEY UPDATE seen_time = VALUES(seen_time)`, + `+ds.dialect.OnDuplicateKey("host_id", "seen_time = VALUES(seen_time)"), insertValues, ) if _, err := tx.ExecContext(ctx, query, insertArgs...); err != nil { @@ -3282,7 +3445,7 @@ SELECT h.policy_updated_at, h.refetch_requested, h.refetch_critical_queries_until, - IF(hdep.host_id AND ISNULL(hdep.deleted_at), true, false) AS dep_assigned_to_fleet + (hdep.host_id IS NOT NULL AND hdep.deleted_at IS NULL) AS dep_assigned_to_fleet FROM hosts h LEFT OUTER JOIN @@ -3419,7 +3582,7 @@ func (ds *Datastore) HostByIdentifier(ctx context.Context, identifier string) (* return nil, ctxerr.Wrap(ctx, err, "get host by identifier") } - packStats, err := loadHostPackStatsDB(ctx, ds.reader(ctx), host.ID, host.Platform) + packStats, err := loadHostPackStatsDB(ctx, ds.reader(ctx), host.ID, host.Platform, ds.dialect) if err != nil { return nil, err } @@ -3446,7 +3609,7 @@ func (ds *Datastore) AddHostsToTeam(ctx context.Context, params *fleet.AddHostsT hostIDsBatch := hostIDs[start:end] err := ds.withRetryTxx( ctx, func(tx sqlx.ExtContext) error { - if err := cleanupPolicyMembershipOnTeamChange(ctx, tx, hostIDsBatch); err != nil { + if err := cleanupPolicyMembershipOnTeamChange(ctx, tx, ds.dialect, hostIDsBatch); err != nil { return ctxerr.Wrap(ctx, err, "AddHostsToTeam delete policy membership") } if err := cleanupQueryResultsOnTeamChange(ctx, tx, hostIDsBatch); err != nil { @@ -3483,14 +3646,14 @@ func (ds *Datastore) AddHostsToTeam(ctx context.Context, params *fleet.AddHostsT } func (ds *Datastore) SaveHostAdditional(ctx context.Context, hostID uint, additional *json.RawMessage) error { - return saveHostAdditionalDB(ctx, ds.writer(ctx), hostID, additional) + return saveHostAdditionalDB(ctx, ds.writer(ctx), ds.dialect, hostID, additional) } -func saveHostAdditionalDB(ctx context.Context, exec sqlx.ExecerContext, hostID uint, additional *json.RawMessage) error { +func saveHostAdditionalDB(ctx context.Context, exec sqlx.ExecerContext, dialect DialectHelper, hostID uint, additional *json.RawMessage) error { sql := ` INSERT INTO host_additional (host_id, additional) VALUES (?, ?) - ON DUPLICATE KEY UPDATE additional = VALUES(additional) + ` + dialect.OnDuplicateKey("host_id", "additional = VALUES(additional)") + ` ` if _, err := exec.ExecContext(ctx, sql, hostID, additional); err != nil { return ctxerr.Wrap(ctx, err, "insert additional") @@ -3500,11 +3663,11 @@ func saveHostAdditionalDB(ctx context.Context, exec sqlx.ExecerContext, hostID u func (ds *Datastore) SaveHostUsers(ctx context.Context, hostID uint, users []fleet.HostUser) error { return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return saveHostUsersDB(ctx, tx, hostID, users) + return saveHostUsersDB(ctx, tx, ds.dialect, hostID, users) }) } -func saveHostUsersDB(ctx context.Context, tx sqlx.ExtContext, hostID uint, users []fleet.HostUser) error { +func saveHostUsersDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, hostID uint, users []fleet.HostUser) error { currentHostUsers, err := loadHostUsersDB(ctx, tx, hostID) if err != nil { return err @@ -3529,11 +3692,11 @@ func saveHostUsersDB(ctx context.Context, tx sqlx.ExtContext, hostID uint, users insertSql := fmt.Sprintf( `INSERT INTO host_users (host_id, uid, username, user_type, groupname, shell) VALUES %s - ON DUPLICATE KEY UPDATE + `+dialect.OnDuplicateKey("host_id,uid,username", ` user_type = VALUES(user_type), groupname = VALUES(groupname), shell = VALUES(shell), - removed_at = NULL`, + removed_at = NULL`), insertValues, ) if _, err := tx.ExecContext(ctx, insertSql, insertArgs...); err != nil { @@ -3636,12 +3799,12 @@ func (ds *Datastore) ListPoliciesForHost(ctx context.Context, host *fleet.Host) // We log to help troubleshooting in case this happens. ds.logger.ErrorContext(ctx, "unrecognized platform", "hostID", host.ID, "platform", host.Platform) } - query := `SELECT p.id, p.team_id, p.resolution, p.name, p.query, p.description, p.author_id, p.platforms, p.critical, p.created_at, p.updated_at, p.conditional_access_enabled, p.type, + query := fmt.Sprintf(`SELECT p.id, p.team_id, p.resolution, p.name, p.query, p.description, p.author_id, p.platforms, p.critical, p.created_at, p.updated_at, p.conditional_access_enabled, p.type, COALESCE(u.name, '') AS author_name, COALESCE(u.email, '') AS author_email, CASE - WHEN pm.passes = 1 THEN 'pass' - WHEN pm.passes = 0 THEN 'fail' + WHEN pm.passes = true THEN 'pass' + WHEN pm.passes = false THEN 'fail' ELSE '' END AS response, coalesce(p.resolution, '') as resolution @@ -3649,14 +3812,14 @@ func (ds *Datastore) ListPoliciesForHost(ctx context.Context, host *fleet.Host) LEFT JOIN policy_membership pm ON (p.id=pm.policy_id AND host_id=?) LEFT JOIN users u ON p.author_id = u.id WHERE (p.team_id IS NULL OR p.team_id = COALESCE((SELECT team_id FROM hosts WHERE id = ?), 0)) - AND (p.platforms IS NULL OR p.platforms = '' OR FIND_IN_SET(?, p.platforms) != 0) + AND (p.platforms IS NULL OR p.platforms = '' OR %s != 0)`, ds.dialect.FindInSet("?", "p.platforms")) + ` AND ( -- Policy has no include labels NOT EXISTS ( SELECT 1 FROM policy_labels pl WHERE pl.policy_id = p.id - AND pl.exclude = 0 + AND pl.exclude = false ) -- Policy is included in the include_any list OR EXISTS ( @@ -3664,7 +3827,7 @@ func (ds *Datastore) ListPoliciesForHost(ctx context.Context, host *fleet.Host) FROM policy_labels pl INNER JOIN label_membership lm ON (lm.host_id = ? AND lm.label_id = pl.label_id) WHERE pl.policy_id = p.id - AND pl.exclude = 0 + AND pl.exclude = false ) ) -- Policy is not included in the exclude_any list @@ -3673,9 +3836,14 @@ func (ds *Datastore) ListPoliciesForHost(ctx context.Context, host *fleet.Host) FROM policy_labels pl INNER JOIN label_membership lm ON (lm.host_id = ? AND lm.label_id = pl.label_id) WHERE pl.policy_id = p.id - AND pl.exclude = 1 + AND pl.exclude = true ) - ORDER BY FIELD(response, 'fail', '', 'pass'), p.name` + ORDER BY CASE + WHEN pm.passes = false THEN 1 + WHEN pm.passes IS NULL THEN 2 + WHEN pm.passes = true THEN 3 + ELSE 0 + END, p.name` var policies []*fleet.HostPolicy if err := sqlx.SelectContext(ctx, ds.reader(ctx), &policies, query, host.ID, host.ID, host.FleetPlatform(), host.ID, host.ID); err != nil { @@ -4296,8 +4464,8 @@ func (ds *Datastore) replaceHostMunkiIssues(ctx context.Context, hostID uint, ms if counts.CountNew < len(newIDs) { // must insert missing IDs - const ( - insStmt = `INSERT INTO host_munki_issues (host_id, munki_issue_id) VALUES %s ON DUPLICATE KEY UPDATE host_id = host_id` + var ( + insStmt = `INSERT INTO host_munki_issues (host_id, munki_issue_id) VALUES %s ` + ds.dialect.OnDuplicateKey("host_id,munki_issue_id", "host_id = VALUES(host_id)") stmtPart = `(?, ?),` ) @@ -4402,9 +4570,9 @@ func (ds *Datastore) getOrInsertMunkiIssues(ctx context.Context, errors, warning // create any missing munki issues (using the primary) if missing := missingIDs(); len(missing) > 0 { - const ( + var ( // UPDATE issue_type = issue_type results in a no-op in mysql (https://stackoverflow.com/a/4596409/1094941) - insStmt = `INSERT INTO munki_issues (name, issue_type) VALUES %s ON DUPLICATE KEY UPDATE issue_type = issue_type` + insStmt = `INSERT INTO munki_issues (name, issue_type) VALUES %s ` + ds.dialect.OnDuplicateKey("name, issue_type", "issue_type = VALUES(issue_type)") stmtParts = `(?, ?),` ) @@ -5136,14 +5304,11 @@ func (ds *Datastore) generateAggregatedMunkiVersion(ctx context.Context, teamID return ctxerr.Wrap(ctx, err, "marshaling stats") } - _, err = ds.writer(ctx).ExecContext(ctx, - ` + _, err = ds.writer(ctx).ExecContext(ctx, ` INSERT INTO aggregated_stats (id, global_stats, type, json_value) VALUES (?, ?, ?, ?) -ON DUPLICATE KEY UPDATE - json_value = VALUES(json_value), - updated_at = CURRENT_TIMESTAMP -`, +`+ds.dialect.OnDuplicateKey("id,type,global_stats", `json_value = VALUES(json_value), + updated_at = CURRENT_TIMESTAMP`), id, globalStats, aggregatedStatsTypeMunkiVersions, versionsJson, ) if err != nil { @@ -5197,10 +5362,9 @@ func (ds *Datastore) generateAggregatedMunkiIssues(ctx context.Context, teamID * _, err = ds.writer(ctx).ExecContext(ctx, ` INSERT INTO aggregated_stats (id, global_stats, type, json_value) VALUES (?, ?, ?, ?) -ON DUPLICATE KEY UPDATE - json_value = VALUES(json_value), - updated_at = CURRENT_TIMESTAMP -`, id, globalStats, aggregatedStatsTypeMunkiIssues, issuesJSON) +`+ds.dialect.OnDuplicateKey("id,type,global_stats", `json_value = VALUES(json_value), + updated_at = CURRENT_TIMESTAMP`), + id, globalStats, aggregatedStatsTypeMunkiIssues, issuesJSON) if err != nil { return ctxerr.Wrapf(ctx, err, "inserting stats for munki_issues id %d", id) } @@ -5213,7 +5377,7 @@ func (ds *Datastore) generateAggregatedMDMStatus(ctx context.Context, teamID *ui globalStats = true status fleet.AggregatedMDMStatus ) - // NOTE: ds.UpdateHostTablesOnMDMUnenroll sets installed_from_dep = 0 so DEP hosts are not counted as pending after unenrollment + // NOTE: ds.UpdateHostTablesOnMDMUnenroll sets installed_from_dep = false so DEP hosts are not counted as pending after unenrollment query := `SELECT COUNT(DISTINCT host_id) as hosts_count, COALESCE(SUM(CASE WHEN NOT enrolled AND NOT installed_from_dep THEN 1 ELSE 0 END), 0) as unenrolled_hosts_count, @@ -5253,14 +5417,11 @@ func (ds *Datastore) generateAggregatedMDMStatus(ctx context.Context, teamID *ui return ctxerr.Wrap(ctx, err, "marshaling stats") } - _, err = ds.writer(ctx).ExecContext(ctx, - ` + _, err = ds.writer(ctx).ExecContext(ctx, ` INSERT INTO aggregated_stats (id, global_stats, type, json_value) VALUES (?, ?, ?, ?) -ON DUPLICATE KEY UPDATE - json_value = VALUES(json_value), - updated_at = CURRENT_TIMESTAMP -`, +`+ds.dialect.OnDuplicateKey("id,type,global_stats", `json_value = VALUES(json_value), + updated_at = CURRENT_TIMESTAMP`), id, globalStats, platformKey(aggregatedStatsTypeMDMStatusPartial, platform), statusJson, ) if err != nil { @@ -5306,7 +5467,7 @@ func (ds *Datastore) generateAggregatedMDMSolutions(ctx context.Context, teamID args = append(args, platform) query += whereAnd + ` h.platform = ? ` } - query += ` GROUP BY id, server_url, name` + query += ` GROUP BY mdms.id, mdms.server_url, mdms.name` err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, query, args...) if err != nil { return ctxerr.Wrapf(ctx, err, "getting aggregated data from host_mdm") @@ -5317,14 +5478,11 @@ func (ds *Datastore) generateAggregatedMDMSolutions(ctx context.Context, teamID return ctxerr.Wrap(ctx, err, "marshaling stats") } - _, err = ds.writer(ctx).ExecContext(ctx, - ` + _, err = ds.writer(ctx).ExecContext(ctx, ` INSERT INTO aggregated_stats (id, global_stats, type, json_value) VALUES (?, ?, ?, ?) -ON DUPLICATE KEY UPDATE - json_value = VALUES(json_value), - updated_at = CURRENT_TIMESTAMP -`, +`+ds.dialect.OnDuplicateKey("id,type,global_stats", `json_value = VALUES(json_value), + updated_at = CURRENT_TIMESTAMP`), id, globalStats, platformKey(aggregatedStatsTypeMDMSolutionsPartial, platform), resultsJSON, ) if err != nil { @@ -5339,7 +5497,7 @@ ON DUPLICATE KEY UPDATE // // If the host doesn't exist, a NotFoundError is returned. func (ds *Datastore) HostLite(ctx context.Context, id uint) (*fleet.Host, error) { - query, args, err := dialect.From(goqu.I("hosts")).Select( + query, args, err := ds.dialect.GoquDialect().From(goqu.I("hosts")).Select( "id", "created_at", "updated_at", @@ -5671,7 +5829,7 @@ func (ds *Datastore) executeOSVersionQuery(ctx context.Context, teamFilter *flee args = append(args, *teamFilter.TeamID, false) case teamFilter != nil: query += " AND " + ds.whereFilterGlobalOrTeamIDByTeamsWithSqlFilter( - *teamFilter, "global_stats = 1 AND id = 0", "global_stats = 0 AND id", + *teamFilter, "global_stats = true AND id = 0", "global_stats = false AND id", ) default: query += " AND id = ? AND global_stats = ?" @@ -5804,7 +5962,7 @@ func (ds *Datastore) UpdateOSVersions(ctx context.Context) error { insertStmt := "INSERT INTO aggregated_stats (id, global_stats, type, json_value) VALUES " insertStmt += strings.TrimSuffix(strings.Repeat("(?,?,?,?),", len(statsByTeamID)+1), ",") // +1 due to global stats - insertStmt += " ON DUPLICATE KEY UPDATE json_value = VALUES(json_value), updated_at = CURRENT_TIMESTAMP" + insertStmt += " " + ds.dialect.OnDuplicateKey("id,type,global_stats", "json_value = VALUES(json_value), updated_at = CURRENT_TIMESTAMP") if _, err := ds.writer(ctx).ExecContext(ctx, insertStmt, args...); err != nil { return ctxerr.Wrapf(ctx, err, "insert os versions into aggregated stats") @@ -5843,7 +6001,7 @@ func (ds *Datastore) HostIDsByOSID( ) ([]uint, error) { var ids []uint - stmt := dialect.From("host_operating_system"). + stmt := ds.dialect.GoquDialect().From("host_operating_system"). Select("host_id"). Where( goqu.C("os_id").Eq(osID)). @@ -5872,7 +6030,7 @@ func (ds *Datastore) HostIDsByOSVersion( ) ([]uint, error) { var ids []uint - stmt := dialect.From("hosts"). + stmt := ds.dialect.GoquDialect().From("hosts"). Select("id"). Where( goqu.C("platform").Eq(osVersion.Platform), @@ -6274,39 +6432,43 @@ func (ds *Datastore) GetHostIssuesLastUpdated(ctx context.Context, hostId uint) func (ds *Datastore) UpdateHostIssuesFailingPolicies(ctx context.Context, hostIDs []uint) error { return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return updateHostIssuesFailingPolicies(ctx, tx, hostIDs) + return updateHostIssuesFailingPolicies(ctx, tx, ds.dialect, hostIDs) }) } func (ds *Datastore) UpdateHostIssuesFailingPoliciesForSingleHost(ctx context.Context, hostID uint) error { var tx sqlx.ExecerContext = ds.writer(ctx) - return updateHostIssuesFailingPoliciesForSingleHost(ctx, tx, hostID) + return updateHostIssuesFailingPoliciesForSingleHost(ctx, tx, ds.dialect, hostID) } -func updateHostIssuesFailingPoliciesForSingleHost(ctx context.Context, tx sqlx.ExecerContext, hostID uint) error { +func updateHostIssuesFailingPoliciesForSingleHost(ctx context.Context, tx sqlx.ExecerContext, dialect DialectHelper, hostID uint) error { + // Use bare column name for critical_vulnerabilities_count — in both MySQL ODKU and PG + // DO UPDATE SET, a bare column reference returns the existing row's value (not the + // inserted/excluded value). VALUES(col) in MySQL ODKU returns DEFAULT when col is not in + // the INSERT list, which would incorrectly zero out the existing count. stmt := ` INSERT INTO host_issues (host_id, failing_policies_count, total_issues_count) - SELECT host_id.id, COALESCE(SUM(!pm.passes), 0), COALESCE(SUM(!pm.passes), 0) + SELECT host_id.id, COALESCE(SUM(CASE WHEN pm.passes = false THEN 1 ELSE 0 END), 0), COALESCE(SUM(CASE WHEN pm.passes = false THEN 1 ELSE 0 END), 0) FROM policy_membership pm - RIGHT JOIN (SELECT ? as id) as host_id + RIGHT JOIN (SELECT CAST(? AS UNSIGNED) as id) as host_id ON pm.host_id = host_id.id GROUP BY host_id.id - ON DUPLICATE KEY UPDATE + ` + dialect.OnDuplicateKey("host_id", ` failing_policies_count = VALUES(failing_policies_count), - total_issues_count = VALUES(failing_policies_count) + critical_vulnerabilities_count` + total_issues_count = VALUES(failing_policies_count) + critical_vulnerabilities_count`) if _, err := tx.ExecContext(ctx, stmt, hostID); err != nil { return ctxerr.Wrap(ctx, err, "updating failing policies in host issues for one host") } return nil } -func updateHostIssuesFailingPolicies(ctx context.Context, tx sqlx.ExecerContext, hostIDs []uint) error { +func updateHostIssuesFailingPolicies(ctx context.Context, tx sqlx.ExecerContext, dialect DialectHelper, hostIDs []uint) error { if len(hostIDs) == 0 { return nil } if len(hostIDs) == 1 { - return updateHostIssuesFailingPoliciesForSingleHost(ctx, tx, hostIDs[0]) + return updateHostIssuesFailingPoliciesForSingleHost(ctx, tx, dialect, hostIDs[0]) } // For multiple hosts, lock policy_membership rows first to prevent deadlocks @@ -6326,15 +6488,24 @@ func updateHostIssuesFailingPolicies(ctx context.Context, tx sqlx.ExecerContext, // Insert/update host_issues entries for hosts that are in policy_membership. // Initially, these two statements were combined into one statement using `SELECT ? AS id UNION ALL` approach to include the host IDs that // were not in policy_membership (similar how the above query for 1 host works). However, in load testing we saw an error: Thread stack overrun: 242191 bytes used of a 262144 byte stack - insertStmt := ` + // PG: !boolean is not valid; use CASE WHEN. + // Use bare column name for critical_vulnerabilities_count — in both MySQL ODKU and PG + // DO UPDATE SET, a bare column reference returns the existing row's value. + var sumExpr string + if dialect.IsPostgres() { + sumExpr = "COALESCE(SUM(CASE WHEN pm.passes = false THEN 1 ELSE 0 END), 0)" + } else { + sumExpr = "COALESCE(SUM(!pm.passes), 0)" + } + insertStmt := fmt.Sprintf(` INSERT INTO host_issues (host_id, failing_policies_count, total_issues_count) - SELECT pm.host_id, COALESCE(SUM(!pm.passes), 0), COALESCE(SUM(!pm.passes), 0) + SELECT pm.host_id, %s, %s FROM policy_membership pm WHERE pm.host_id IN (?) GROUP BY pm.host_id - ON DUPLICATE KEY UPDATE + `, sumExpr, sumExpr) + dialect.OnDuplicateKey("host_id", ` failing_policies_count = VALUES(failing_policies_count), - total_issues_count = VALUES(failing_policies_count) + critical_vulnerabilities_count` + total_issues_count = VALUES(failing_policies_count) + critical_vulnerabilities_count`) // Sort host IDs to ensure consistent lock ordering across all transactions. // This prevents deadlocks when multiple transactions process overlapping sets of hosts. @@ -6471,9 +6642,8 @@ func (ds *Datastore) UpdateHostIssuesVulnerabilities(ctx context.Context) error ) stmt := fmt.Sprintf( `INSERT INTO host_issues (host_id, critical_vulnerabilities_count, total_issues_count) VALUES %s - ON DUPLICATE KEY UPDATE - critical_vulnerabilities_count = VALUES(critical_vulnerabilities_count), - total_issues_count = failing_policies_count + VALUES(critical_vulnerabilities_count)`, + `+ds.dialect.OnDuplicateKey("host_id", `critical_vulnerabilities_count = VALUES(critical_vulnerabilities_count), + total_issues_count = failing_policies_count + VALUES(critical_vulnerabilities_count)`), values, ) args := make([]interface{}, 0, totalToProcess*numberOfArgsPerIssue) diff --git a/server/datastore/mysql/in_house_apps.go b/server/datastore/mysql/in_house_apps.go index 4342f6e09c5..148c8045457 100644 --- a/server/datastore/mysql/in_house_apps.go +++ b/server/datastore/mysql/in_house_apps.go @@ -109,9 +109,9 @@ func (ds *Datastore) insertInHouseAppDB(ctx context.Context, tx sqlx.ExtContext, ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` - res, err := tx.ExecContext(ctx, stmt, args...) + id64, err := insertAndGetIDTx(ctx, tx, ds.dialect, stmt, args...) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { teamName, err := ds.getTeamName(ctx, payload.TeamID) if err != nil { return 0, ctxerr.Wrap(ctx, err) @@ -121,13 +121,9 @@ func (ds *Datastore) insertInHouseAppDB(ctx context.Context, tx sqlx.ExtContext, } return 0, ctxerr.Wrap(ctx, err, "insertInHouseAppDB") } - id64, err := res.LastInsertId() installerID := uint(id64) //nolint:gosec // dismiss G115 - if err != nil { - return 0, ctxerr.Wrap(ctx, err, "insertInHouseAppDB") - } - if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, installerID, *payload.ValidatedLabels, softwareTypeInHouseApp); err != nil { + if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, ds.dialect, installerID, *payload.ValidatedLabels, softwareTypeInHouseApp); err != nil { return 0, ctxerr.Wrap(ctx, err, "insertInHouseAppDB") } @@ -286,7 +282,7 @@ func (ds *Datastore) SaveInHouseAppUpdates(ctx context.Context, payload *fleet.U } if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { teamName, err := ds.getTeamName(ctx, payload.TeamID) if err != nil { return ctxerr.Wrap(ctx, err) @@ -297,7 +293,7 @@ func (ds *Datastore) SaveInHouseAppUpdates(ctx context.Context, payload *fleet.U } if payload.ValidatedLabels != nil { - if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, payload.InstallerID, *payload.ValidatedLabels, softwareTypeInHouseApp); err != nil { + if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, ds.dialect, payload.InstallerID, *payload.ValidatedLabels, softwareTypeInHouseApp); err != nil { return ctxerr.Wrap(ctx, err, "upsert in house app labels") } } @@ -309,7 +305,7 @@ func (ds *Datastore) SaveInHouseAppUpdates(ctx context.Context, payload *fleet.U } if payload.DisplayName != nil { - if err := updateSoftwareTitleDisplayName(ctx, tx, payload.TeamID, payload.TitleID, *payload.DisplayName); err != nil { + if err := updateSoftwareTitleDisplayName(ctx, tx, ds.dialect, payload.TeamID, payload.TitleID, *payload.DisplayName); err != nil { return ctxerr.Wrap(ctx, err, "update in house app display name") } } @@ -528,7 +524,7 @@ VALUES } err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - res, err := tx.ExecContext(ctx, insertUAStmt, + activityID, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertUAStmt, hostID, opts.Priority(), userID, @@ -540,8 +536,6 @@ VALUES if err != nil { return ctxerr.Wrap(ctx, err, "insert in house app install request") } - - activityID, _ := res.LastInsertId() _, err = tx.ExecContext(ctx, insertIHAUAStmt, activityID, inHouseAppID, @@ -734,17 +728,17 @@ WHERE } func (ds *Datastore) BatchSetInHouseAppsInstallers(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { - const upsertSoftwareTitles = ` + upsertSoftwareTitles := ` INSERT INTO software_titles (name, source, extension_for, bundle_identifier) VALUES %s -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("id", ` name = VALUES(name), source = VALUES(source), extension_for = VALUES(extension_for), bundle_identifier = VALUES(bundle_identifier) -` +`) const loadSoftwareTitles = ` SELECT @@ -916,7 +910,7 @@ WHERE title_id = ? ` - const insertNewOrEditedInstaller = ` + insertNewOrEditedInstaller := ` INSERT INTO in_house_apps ( title_id, team_id, @@ -931,7 +925,7 @@ INSERT INTO in_house_apps ( ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("id", ` filename = VALUES(filename), version = VALUES(version), storage_id = VALUES(storage_id), @@ -939,7 +933,7 @@ ON DUPLICATE KEY UPDATE bundle_identifier = VALUES(bundle_identifier), self_service = VALUES(self_service), url = VALUES(url) -` +`) const loadInHouseInstallerID = ` SELECT @@ -968,7 +962,7 @@ WHERE in_house_app_id = ? ` - const upsertInHouseLabels = ` + upsertInHouseLabels := ` INSERT INTO in_house_app_labels ( in_house_app_id, @@ -978,10 +972,10 @@ INSERT INTO ) VALUES %s -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("id", ` exclude = VALUES(exclude), require_all = VALUES(require_all) -` +`) const loadExistingInHouseLabels = ` SELECT @@ -1009,8 +1003,7 @@ WHERE software_category_id NOT IN (?) ` - const upsertInHouseCategories = ` -INSERT IGNORE INTO + const upsertInHouseCategoriesSuffix = ` in_house_app_software_categories ( in_house_app_id, software_category_id @@ -1396,7 +1389,7 @@ WHERE upsertCategoriesArgs = append(upsertCategoriesArgs, installerID, catID) } upsertCategoriesValues := strings.TrimSuffix(strings.Repeat("(?,?),", len(installer.CategoryIDs)), ",") - _, err = tx.ExecContext(ctx, fmt.Sprintf(upsertInHouseCategories, upsertCategoriesValues), upsertCategoriesArgs...) + _, err = tx.ExecContext(ctx, ds.dialect.InsertIgnoreInto()+fmt.Sprintf(upsertInHouseCategoriesSuffix, upsertCategoriesValues)+ds.dialect.OnConflictDoNothing("in_house_app_id,software_category_id"), upsertCategoriesArgs...) if err != nil { return ctxerr.Wrapf(ctx, err, "insert new/edited categories for in-house with name %q", installer.Filename) } @@ -1405,7 +1398,7 @@ WHERE // update display name for the software title if it needs to be updated or inserted // no deletions will happen, display names will be set to empty if needed if name, ok := displayNameIDMap[titleID]; (ok && name != installer.DisplayName) || (!ok && installer.DisplayName != "") { - if err := updateSoftwareTitleDisplayName(ctx, tx, tmID, titleID, installer.DisplayName); err != nil { + if err := updateSoftwareTitleDisplayName(ctx, tx, ds.dialect, tmID, titleID, installer.DisplayName); err != nil { return ctxerr.Wrapf(ctx, err, "update software title display name for in-house app with name %q", installer.Filename) } } diff --git a/server/datastore/mysql/invites.go b/server/datastore/mysql/invites.go index e3305c8a0b8..c11d9fb61b5 100644 --- a/server/datastore/mysql/invites.go +++ b/server/datastore/mysql/invites.go @@ -37,15 +37,14 @@ func (ds *Datastore) NewInvite(ctx context.Context, i *fleet.Invite) (*fleet.Inv VALUES ( ?, ?, ?, ?, ?, ?, ?, ?) ` - result, err := tx.ExecContext(ctx, sqlStmt, i.InvitedBy, i.Email, + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, sqlStmt, i.InvitedBy, i.Email, i.Name, i.Position, i.Token, i.SSOEnabled, i.MFAEnabled, i.GlobalRole) - if err != nil && IsDuplicate(err) { + if err != nil && ds.dialect.IsDuplicate(err) { return ctxerr.Wrap(ctx, alreadyExists("Invite", i.Email)) } else if err != nil { return ctxerr.Wrap(ctx, err, "create invite") } - id, _ := result.LastInsertId() i.ID = uint(id) //nolint:gosec // dismiss G115 if len(i.Teams) == 0 { diff --git a/server/datastore/mysql/jobs.go b/server/datastore/mysql/jobs.go index 159b8defa9a..b453912e2dc 100644 --- a/server/datastore/mysql/jobs.go +++ b/server/datastore/mysql/jobs.go @@ -26,12 +26,11 @@ VALUES (?, ?, ?, ?, ?, COALESCE(?, NOW())) if !job.NotBefore.IsZero() { notBefore = &job.NotBefore } - result, err := ds.writer(ctx).ExecContext(ctx, query, job.Name, job.Args, job.State, job.Retries, job.Error, notBefore) + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), query, job.Name, job.Args, job.State, job.Retries, job.Error, notBefore) if err != nil { return nil, err } - id, _ := result.LastInsertId() job.ID = uint(id) //nolint:gosec // dismiss G115 return job, nil diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index ebd63c33d7b..69e578bd93b 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -190,7 +190,7 @@ func (ds *Datastore) ApplyLabelSpecsWithAuthor(ctx context.Context, specs []*fle } } - sql := ` + insertSQL := ` INSERT INTO labels ( name, description, @@ -202,7 +202,7 @@ func (ds *Datastore) ApplyLabelSpecsWithAuthor(ctx context.Context, specs []*fle author_id, team_id ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ? ) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("name", ` name = VALUES(name), description = VALUES(description), query = VALUES(query), @@ -210,23 +210,13 @@ func (ds *Datastore) ApplyLabelSpecsWithAuthor(ctx context.Context, specs []*fle label_type = VALUES(label_type), label_membership_type = VALUES(label_membership_type), criteria = VALUES(criteria) - ` - - prepTx, ok := tx.(sqlx.PreparerContext) - if !ok { - return ctxerr.New(ctx, "tx in ApplyLabelSpecs is not a sqlx.PreparerContext") - } - stmt, err := prepTx.PrepareContext(ctx, sql) - if err != nil { - return ctxerr.Wrap(ctx, err, "prepare ApplyLabelSpecs insert") - } - defer stmt.Close() + `) for _, s := range specs { if s.Name == "" { return ctxerr.New(ctx, "label name must not be empty") } - insertLabelResult, err := stmt.ExecContext(ctx, s.Name, s.Description, s.Query, s.Platform, s.LabelType, s.LabelMembershipType, s.HostVitalsCriteria, authorID, s.TeamID) + insertedID, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertSQL, s.Name, s.Description, s.Query, s.Platform, s.LabelType, s.LabelMembershipType, s.HostVitalsCriteria, authorID, s.TeamID) if err != nil { return ctxerr.Wrap(ctx, err, "exec ApplyLabelSpecs insert") } @@ -262,18 +252,12 @@ func (ds *Datastore) ApplyLabelSpecsWithAuthor(ctx context.Context, specs []*fle // Use the existing label ID labelID = existing.ID } else { - // New label - fetch the ID we just created - id, err := insertLabelResult.LastInsertId() - if err != nil { - return ctxerr.Wrap(ctx, err, "get new label ID for manual membership") - } - labelID = uint(id) //nolint:gosec + // New label - use the ID from the insert + labelID = uint(insertedID) //nolint:gosec } - sql = ` -DELETE FROM label_membership WHERE label_id = ? -` - _, err = tx.ExecContext(ctx, sql, labelID) + delSQL := `DELETE FROM label_membership WHERE label_id = ?` + _, err = tx.ExecContext(ctx, delSQL, labelID) if err != nil { return ctxerr.Wrap(ctx, err, "clear membership for ID") } @@ -323,15 +307,15 @@ DELETE FROM label_membership WHERE label_id = ? // Use ignore because duplicate hostnames could appear in // different batches and would result in duplicate key errors. - sql = fmt.Sprintf( - `INSERT IGNORE INTO label_membership (label_id, host_id) (SELECT DISTINCT ?, id FROM hosts WHERE %s)`, + memberSQL := fmt.Sprintf( + ds.dialect.InsertIgnoreInto()+` label_membership (label_id, host_id) (SELECT DISTINCT ?, id FROM hosts WHERE %s)`+ds.dialect.OnConflictDoNothing("host_id,label_id"), hostsFilterClause, ) - sql, args, err := sqlx.In(sql, labelID, stringIdents, stringIdents, stringIdents, intIdents) + memberSQL, args, err := sqlx.In(memberSQL, labelID, stringIdents, stringIdents, stringIdents, intIdents) if err != nil { return ctxerr.Wrap(ctx, err, "build membership IN statement") } - _, err = tx.ExecContext(ctx, sql, args...) + _, err = tx.ExecContext(ctx, memberSQL, args...) if err != nil { return ctxerr.Wrap(ctx, err, "execute membership INSERT") } @@ -429,9 +413,8 @@ func (ds *Datastore) UpdateLabelMembershipByHostIDs(ctx context.Context, label f } // Build the final SQL query with the dynamically generated placeholders - sql := ` -INSERT IGNORE INTO label_membership (label_id, host_id) -VALUES ` + strings.Join(placeholders, ", ") + sql := ds.dialect.InsertIgnoreInto() + ` label_membership (label_id, host_id) +VALUES ` + strings.Join(placeholders, ", ") + ds.dialect.OnConflictDoNothing("host_id,label_id") sql, args, err := sqlx.In(sql, values...) if err != nil { return ctxerr.Wrap(ctx, err, "build membership IN statement") @@ -482,7 +465,7 @@ func (ds *Datastore) UpdateLabelMembershipByHostCriteria(ctx context.Context, hv err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { // Insert new label membership based on the label query. - sql := fmt.Sprintf(`INSERT INTO label_membership (label_id, host_id) SELECT candidate.label_id, candidate.host_id FROM (%s) as candidate ON DUPLICATE KEY UPDATE host_id = label_membership.host_id`, labelQuery) + sql := fmt.Sprintf(`INSERT INTO label_membership (label_id, host_id) SELECT candidate.label_id, candidate.host_id FROM (%s) as candidate `+ds.dialect.OnDuplicateKey("host_id,label_id", `host_id = label_membership.host_id`), labelQuery) _, err := tx.ExecContext(ctx, sql, queryVals...) if err != nil { return ctxerr.Wrap(ctx, err, "execute membership INSERT") @@ -620,9 +603,7 @@ func (ds *Datastore) NewLabel(ctx context.Context, label *fleet.Label, opts ...f team_id ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ? ) ` - result, err := ds.writer(ctx).ExecContext( - ctx, - query, + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), query, label.Name, label.Description, label.Query, @@ -637,7 +618,6 @@ func (ds *Datastore) NewLabel(ctx context.Context, label *fleet.Label, opts ...f return nil, ctxerr.Wrap(ctx, err, "inserting label") } - id, _ := result.LastInsertId() label.ID = uint(id) //nolint:gosec // dismiss G115 now := time.Now().UTC().Truncate(time.Second) label.CreatedAt = now @@ -680,7 +660,7 @@ func (ds *Datastore) DeleteLabel(ctx context.Context, name string, filter fleet. return ctxerr.Wrap(ctx, err, "getting label id to delete") } if err := deleteLabelsInTx(ctx, tx, []uint{labelID}); err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { return ctxerr.Wrap(ctx, foreignKey("labels", name), "delete label") } return ctxerr.Wrap(ctx, err, "delete labels in tx") @@ -933,7 +913,7 @@ func (ds *Datastore) RecordLabelQueryExecutions(ctx context.Context, host *fleet // Complete inserts if necessary if len(vals) > 0 { sql := `INSERT INTO label_membership (updated_at, label_id, host_id) VALUES ` - sql += strings.Join(bindvars, ",") + ` ON DUPLICATE KEY UPDATE updated_at = VALUES(updated_at)` + sql += strings.Join(bindvars, ",") + ` ` + ds.dialect.OnDuplicateKey("host_id,label_id", `updated_at = VALUES(updated_at)`) _, err := tx.ExecContext(ctx, sql, vals...) if err != nil { @@ -996,8 +976,8 @@ func (ds *Datastore) RecordLabelQueryExecutions(ctx context.Context, host *fleet func (ds *Datastore) ListLabelsForHost(ctx context.Context, hid uint) ([]*fleet.Label, error) { sqlStatement := ` SELECT labels.* from labels JOIN label_membership lm + ON lm.label_id = labels.id WHERE lm.host_id = ? - AND lm.label_id = labels.id ` labels := []*fleet.Label{} @@ -1101,11 +1081,11 @@ func (ds *Datastore) ListHostsInLabel(ctx context.Context, filter fleet.TeamFilt deviceMappingJoin := fmt.Sprintf(`LEFT JOIN ( SELECT host_id, - CONCAT('[', GROUP_CONCAT(JSON_OBJECT('email', email, 'source', %s)), ']') AS device_mapping + CONCAT('[', %s, ']') AS device_mapping FROM host_emails GROUP BY - host_id) dm ON dm.host_id = h.id`, deviceMappingTranslateSourceColumn("")) + host_id) dm ON dm.host_id = h.id`, ds.dialect.GroupConcat(fmt.Sprintf("JSON_OBJECT('email', email, 'source', %s)", deviceMappingTranslateSourceColumn("")), ",")) if !opt.DeviceMapping { deviceMappingJoin = "" } @@ -1282,10 +1262,11 @@ func (ds *Datastore) searchLabelsWithOmits(ctx context.Context, filter fleet.Tea ) AS host_count FROM labels l WHERE ( - MATCH(l.name) AGAINST(? IN BOOLEAN MODE) + %s ) AND l.id NOT IN (?) `, ds.whereFilterHostsByTeams(filter, "h"), + ds.dialect.FullTextMatch([]string{"l.name"}, "?"), ) sql, args, err := applyLabelTeamFilter(sqlStatement, filter, transformQuery(query), omit) @@ -1413,9 +1394,10 @@ func (ds *Datastore) SearchLabels(ctx context.Context, filter fleet.TeamFilter, ) AS host_count FROM labels l WHERE ( - MATCH(name) AGAINST(? IN BOOLEAN MODE) + %s ) `, ds.whereFilterHostsByTeams(filter, "h"), + ds.dialect.FullTextMatch([]string{"name"}, "?"), ) sql, args, err := applyLabelTeamFilter(sql, filter, transformQuery(query)) @@ -1492,7 +1474,7 @@ func (ds *Datastore) AsyncBatchInsertLabelMembership(ctx context.Context, batch sql := `INSERT INTO label_membership (label_id, host_id) VALUES ` sql += strings.Repeat(`(?, ?),`, len(batch)) sql = strings.TrimSuffix(sql, ",") - sql += ` ON DUPLICATE KEY UPDATE updated_at = VALUES(updated_at)` + sql += ` ` + ds.dialect.OnDuplicateKey("host_id,label_id", `updated_at = VALUES(updated_at)`) vals := make([]interface{}, 0, len(batch)*2) for _, tup := range batch { @@ -1510,7 +1492,18 @@ func (ds *Datastore) AsyncBatchDeleteLabelMembership(ctx context.Context, batch // NOTE: this is tested via the server/service/async package tests. rest := strings.Repeat(`UNION ALL SELECT ?, ? `, len(batch)-1) - sql := fmt.Sprintf(` + var sql string + if ds.dialect.IsPostgres() { + sql = fmt.Sprintf(` + DELETE FROM + label_membership + USING + (SELECT ?::integer AS label_id, ?::integer AS host_id %s) del_list + WHERE + label_membership.label_id = del_list.label_id AND + label_membership.host_id = del_list.host_id`, strings.ReplaceAll(rest, "SELECT ?, ?", "SELECT ?::integer, ?::integer")) + } else { + sql = fmt.Sprintf(` DELETE lm FROM @@ -1520,6 +1513,7 @@ func (ds *Datastore) AsyncBatchDeleteLabelMembership(ctx context.Context, batch ON lm.label_id = del_list.label_id AND lm.host_id = del_list.host_id`, rest) + } vals := make([]interface{}, 0, len(batch)*2) for _, tup := range batch { @@ -1612,7 +1606,7 @@ func (ds *Datastore) AddLabelsToHost(ctx context.Context, hostID uint, labelIDs sql := `INSERT INTO label_membership (host_id, label_id) VALUES ` sql += strings.Repeat(`(?, ?),`, len(labelIDs)) sql = strings.TrimSuffix(sql, ",") - sql += ` ON DUPLICATE KEY UPDATE updated_at = NOW()` + sql += ` ` + ds.dialect.OnDuplicateKey("host_id,label_id", `updated_at = NOW()`) args := make([]interface{}, 0, len(labelIDs)*2) for _, labelID := range labelIDs { args = append(args, hostID, labelID) diff --git a/server/datastore/mysql/locks.go b/server/datastore/mysql/locks.go index 52362e8c5b0..263e7713866 100644 --- a/server/datastore/mysql/locks.go +++ b/server/datastore/mysql/locks.go @@ -37,7 +37,7 @@ func (ds *Datastore) Lock(ctx context.Context, name string, owner string, expira func (ds *Datastore) createLock(ctx context.Context, name string, owner string, expiration time.Duration) (sql.Result, error) { return ds.writer(ctx).ExecContext(ctx, - `INSERT IGNORE INTO locks (name, owner, expires_at) VALUES (?, ?, ?)`, + ds.dialect.InsertIgnoreInto()+` locks (name, owner, expires_at) VALUES (?, ?, ?)`+ds.dialect.OnConflictDoNothing("name"), name, owner, time.Now().Add(expiration), ) } diff --git a/server/datastore/mysql/maintained_apps.go b/server/datastore/mysql/maintained_apps.go index 6b593b01e5d..6205b82a752 100644 --- a/server/datastore/mysql/maintained_apps.go +++ b/server/datastore/mysql/maintained_apps.go @@ -12,62 +12,25 @@ import ( ) func (ds *Datastore) UpsertMaintainedApp(ctx context.Context, app *fleet.MaintainedApp) (*fleet.MaintainedApp, error) { - const upsertStmt = ` + upsertStmt := ` INSERT INTO fleet_maintained_apps (name, slug, platform, unique_identifier) VALUES (?, ?, ?, ?) -ON DUPLICATE KEY UPDATE - name = VALUES(name), +` + ds.dialect.OnDuplicateKey("id", `name = VALUES(name), platform = VALUES(platform), - unique_identifier = VALUES(unique_identifier) -` + unique_identifier = VALUES(unique_identifier)`) var appID uint err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { var err error // upsert the maintained app - res, err := tx.ExecContext(ctx, upsertStmt, app.Name, app.Slug, app.Platform, app.UniqueIdentifier) + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, upsertStmt, app.Name, app.Slug, app.Platform, app.UniqueIdentifier) if err != nil { return ctxerr.Wrap(ctx, err, "upsert maintained app") } - id, _ := res.LastInsertId() appID = uint(id) //nolint:gosec // dismiss G115 - - // For darwin apps, update existing software_titles and software entries - // to use the FMA canonical name. This ensures consistency when an FMA - // is added for software that was previously ingested with osquery-reported names. - // - // We only run these UPDATEs when the FMA was actually inserted or modified. - // MySQL's ON DUPLICATE KEY UPDATE returns RowsAffected: - // 0 = duplicate key, no changes (existing FMA with same values) - // 1 = new row inserted - // 2 = duplicate key, values changed - // Skip if RowsAffected == 0 since nothing changed. - rowsAffected, _ := res.RowsAffected() - if app.Platform == "darwin" && app.UniqueIdentifier != "" && rowsAffected > 0 { - _, err = tx.ExecContext(ctx, ` - UPDATE software_titles - SET name = ? - WHERE bundle_identifier = ? - AND name != ? - `, app.Name, app.UniqueIdentifier, app.Name) - if err != nil { - return ctxerr.Wrap(ctx, err, "update software_titles names for FMA") - } - - _, err = tx.ExecContext(ctx, ` - UPDATE software - SET name = ? - WHERE bundle_identifier = ? - AND name != ? - `, app.Name, app.UniqueIdentifier, app.Name) - if err != nil { - return ctxerr.Wrap(ctx, err, "update software names for FMA") - } - } - return nil }) if err != nil { @@ -214,30 +177,6 @@ func (ds *Datastore) ListAvailableFleetMaintainedApps(ctx context.Context, teamI return avail, meta, nil } -func (ds *Datastore) GetFMANamesByIdentifier(ctx context.Context) (map[string]string, error) { - query := `SELECT unique_identifier, name FROM fleet_maintained_apps WHERE platform = 'darwin'` - - rows, err := ds.reader(ctx).QueryContext(ctx, query) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "query FMA names by identifier") - } - defer rows.Close() - - result := make(map[string]string) - for rows.Next() { - var identifier, name string - if err := rows.Scan(&identifier, &name); err != nil { - return nil, ctxerr.Wrap(ctx, err, "scan FMA name row") - } - result[identifier] = name - } - if err := rows.Err(); err != nil { - return nil, ctxerr.Wrap(ctx, err, "iterate FMA name rows") - } - - return result, nil -} - func (ds *Datastore) ClearRemovedFleetMaintainedApps(ctx context.Context, slugsToKeep []string) error { stmt := `DELETE FROM fleet_maintained_apps WHERE slug NOT IN (?)` @@ -260,3 +199,26 @@ func (ds *Datastore) ClearRemovedFleetMaintainedApps(ctx context.Context, slugsT return nil } + +func (ds *Datastore) GetFMANamesByIdentifier(ctx context.Context) (map[string]string, error) { + query := `SELECT unique_identifier, name FROM fleet_maintained_apps WHERE platform = 'darwin'` + + rows, err := ds.reader(ctx).QueryContext(ctx, query) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "query FMA names by identifier") + } + defer rows.Close() + + result := make(map[string]string) + for rows.Next() { + var identifier, name string + if err := rows.Scan(&identifier, &name); err != nil { + return nil, ctxerr.Wrap(ctx, err, "scan FMA name row") + } + result[identifier] = name + } + if err := rows.Err(); err != nil { + return nil, ctxerr.Wrap(ctx, err, "iterating FMA name rows") + } + return result, nil +} diff --git a/server/datastore/mysql/mdm.go b/server/datastore/mysql/mdm.go index c64b0079fd4..c018eae77ce 100644 --- a/server/datastore/mysql/mdm.go +++ b/server/datastore/mysql/mdm.go @@ -1156,8 +1156,8 @@ WHERE GROUP BY profile_uuid, name, syncml HAVING - count_profile_labels > 0 AND - count_host_labels = count_profile_labels + COUNT(*) > 0 AND + COUNT(lm.label_id) = COUNT(*) UNION @@ -1183,9 +1183,9 @@ GROUP BY profile_uuid, name, syncml HAVING -- considers only the profiles with labels, without any broken label, and with the host not in any label - count_profile_labels > 0 AND - count_profile_labels = count_non_broken_labels AND - count_host_labels = 0 + COUNT(*) > 0 AND + COUNT(*) = COUNT(mcpl.label_id) AND + COUNT(lm.label_id) = 0 UNION @@ -1209,8 +1209,8 @@ WHERE GROUP BY profile_uuid, name, syncml HAVING - count_profile_labels > 0 AND - count_host_labels > 0 + COUNT(*) > 0 AND + COUNT(lm.label_id) > 0 ` var profiles []*fleet.ExpectedMDMProfile err := sqlx.SelectContext(ctx, ds.reader(ctx), &profiles, stmt, teamID, hostID, teamID, hostID, teamID, hostID, teamID) @@ -1289,8 +1289,8 @@ WHERE GROUP BY profile_uuid, identifier HAVING - count_profile_labels > 0 AND - count_host_labels = count_profile_labels + COUNT(*) > 0 AND + COUNT(lm.label_id) = COUNT(*) UNION @@ -1323,9 +1323,9 @@ GROUP BY profile_uuid, identifier HAVING -- considers only the profiles with labels, without any broken label, and with the host not in any label - count_profile_labels > 0 AND - count_profile_labels = count_non_broken_labels AND - count_host_labels = 0 + COUNT(*) > 0 AND + COUNT(*) = COUNT(mcpl.label_id) AND + COUNT(lm.label_id) = 0 UNION @@ -1356,8 +1356,8 @@ WHERE GROUP BY profile_uuid, identifier HAVING - count_profile_labels > 0 AND - count_host_labels > 0 + COUNT(*) > 0 AND + COUNT(lm.label_id) > 0 ` var rows []*fleet.ExpectedMDMProfile @@ -1476,6 +1476,7 @@ WHERE func batchSetProfileLabelAssociationsDB( ctx context.Context, tx sqlx.ExtContext, + dialect DialectHelper, profileLabels []fleet.ConfigurationProfileLabel, profileUUIDsWithoutLabels []string, platform string, @@ -1522,10 +1523,10 @@ func batchSetProfileLabelAssociationsDB( (%s_profile_uuid, label_id, label_name, exclude, require_all) VALUES %s - ON DUPLICATE KEY UPDATE + ` + dialect.OnDuplicateKey("id", ` label_id = VALUES(label_id), exclude = VALUES(exclude), - require_all = VALUES(require_all) + require_all = VALUES(require_all)`) + ` ` selectStmt := ` @@ -1674,7 +1675,7 @@ func (ds *Datastore) MDMInsertEULA(ctx context.Context, eula *fleet.MDMEULA) err _, err := ds.writer(ctx).ExecContext(ctx, stmt, eula.Name, eula.Bytes, eula.Token, eula.Sha256) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return ctxerr.Wrap(ctx, alreadyExists("MDMEULA", eula.Token)) } return ctxerr.Wrap(ctx, err, "create EULA") @@ -1779,9 +1780,9 @@ func (ds *Datastore) SetCommandForPendingSCEPRenewal(ctx context.Context, assocs stmt := fmt.Sprintf(` INSERT INTO nano_cert_auth_associations (id, sha256, renew_command_uuid) VALUES %s - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("id,sha256", ` renew_command_uuid = VALUES(renew_command_uuid) - `, strings.TrimSuffix(sb.String(), ",")) + `), strings.TrimSuffix(sb.String(), ",")) return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { res, err := tx.ExecContext(ctx, stmt, args...) @@ -1978,6 +1979,7 @@ func (ds *Datastore) IsHostConnectedToFleetMDM(ctx context.Context, host *fleet. func batchSetProfileVariableAssociationsDB( ctx context.Context, tx sqlx.ExtContext, + dialect DialectHelper, profileVariablesByUUID []fleet.MDMProfileUUIDFleetVariables, platform string, ) (didUpdate bool, err error) { @@ -2080,9 +2082,8 @@ func batchSetProfileVariableAssociationsDB( fleet_variable_id ) VALUES %s - ON DUPLICATE KEY UPDATE - fleet_variable_id = VALUES(fleet_variable_id) - `, platformPrefix, strings.TrimSuffix(valuePart, ",")) + `, platformPrefix, strings.TrimSuffix(valuePart, ",")) + + dialect.OnDuplicateKey("id", "fleet_variable_id = VALUES(fleet_variable_id)") _, err := tx.ExecContext(ctx, stmt, args...) return err @@ -2528,7 +2529,7 @@ func (ds *Datastore) batchSetLabelAndVariableAssociations(ctx context.Context, t } var didUpdateLabels bool - if didUpdateLabels, err = batchSetProfileLabelAssociationsDB(ctx, tx, incomingLabels, profsWithoutLabels, + if didUpdateLabels, err = batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, incomingLabels, profsWithoutLabels, platform); err != nil { return false, ctxerr.Wrap(ctx, err, fmt.Sprintf("inserting %s profile label associations", platform)) } @@ -2554,7 +2555,7 @@ func (ds *Datastore) batchSetLabelAndVariableAssociations(ctx context.Context, t if len(profilesVarsToUpsert) > 0 { var didUpdateVariableAssociations bool - if didUpdateVariableAssociations, err = batchSetProfileVariableAssociationsDB(ctx, tx, profilesVarsToUpsert, platform); err != nil { + if didUpdateVariableAssociations, err = batchSetProfileVariableAssociationsDB(ctx, tx, ds.dialect, profilesVarsToUpsert, platform); err != nil { return false, ctxerr.Wrap(ctx, err, fmt.Sprintf("inserting %s profile variable associations", platform)) } @@ -2706,14 +2707,17 @@ func getMDMIdPAccountByHostID(ctx context.Context, q sqlx.QueryerContext, logger func (ds *Datastore) CleanUpMDMManagedCertificates(ctx context.Context) error { _, err := ds.writer(ctx).ExecContext(ctx, ` - DELETE hmmc FROM host_mdm_managed_certificates hmmc -LEFT JOIN host_mdm_apple_profiles hmap ON hmmc.host_uuid = hmap.host_uuid - AND hmmc.profile_uuid = hmap.profile_uuid -LEFT JOIN host_mdm_windows_profiles hwmp ON hmmc.host_uuid = hwmp.host_uuid - AND hmmc.profile_uuid = hwmp.profile_uuid -WHERE - hmap.host_uuid IS NULL - AND hwmp.host_uuid IS NULL`) + DELETE FROM host_mdm_managed_certificates +WHERE NOT EXISTS ( + SELECT 1 FROM host_mdm_apple_profiles hmap + WHERE hmap.host_uuid = host_mdm_managed_certificates.host_uuid + AND hmap.profile_uuid = host_mdm_managed_certificates.profile_uuid +) +AND NOT EXISTS ( + SELECT 1 FROM host_mdm_windows_profiles hwmp + WHERE hwmp.host_uuid = host_mdm_managed_certificates.host_uuid + AND hwmp.profile_uuid = host_mdm_managed_certificates.profile_uuid +)`) if err != nil { return ctxerr.Wrap(ctx, err, "clean up mdm certificate profiles") } @@ -2738,13 +2742,13 @@ func (ds *Datastore) BulkUpsertMDMManagedCertificates(ctx context.Context, paylo serial ) VALUES %s - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("host_uuid,profile_uuid,ca_name", ` challenge_retrieved_at = VALUES(challenge_retrieved_at), not_valid_before = VALUES(not_valid_before), not_valid_after = VALUES(not_valid_after), type = VALUES(type), ca_name = VALUES(ca_name), - serial = VALUES(serial)`, + serial = VALUES(serial)`), strings.TrimSuffix(valuePart, ","), ) @@ -2832,10 +2836,14 @@ func (ds *Datastore) RenewMDMManagedCertificates(ctx context.Context) error { ON hmmc.host_uuid = hp.host_uuid AND hmmc.profile_uuid = hp.profile_uuid WHERE hmmc.type = ? AND hp.status IS NOT NULL AND hp.operation_type = ? - HAVING - validity_period IS NOT NULL AND - ((validity_period > 30 AND not_valid_after < DATE_ADD(NOW(), INTERVAL 30 DAY)) OR - (validity_period <= 30 AND not_valid_after < DATE_ADD(NOW(), INTERVAL validity_period/2 DAY))) + AND DATEDIFF(hmmc.not_valid_after, hmmc.not_valid_before) IS NOT NULL + AND ( + (DATEDIFF(hmmc.not_valid_after, hmmc.not_valid_before) > 30 + AND hmmc.not_valid_after < DATE_ADD(NOW(), INTERVAL 30 DAY)) + OR + (DATEDIFF(hmmc.not_valid_after, hmmc.not_valid_before) <= 30 + AND hmmc.not_valid_after < DATE_ADD(NOW(), INTERVAL DATEDIFF(hmmc.not_valid_after, hmmc.not_valid_before)/2 DAY)) + ) LIMIT ?`, hostCertType, fleet.MDMOperationTypeInstall, limit) if err != nil { return ctxerr.Wrap(ctx, err, "retrieving mdm managed certificates to renew") diff --git a/server/datastore/mysql/microsoft_mdm.go b/server/datastore/mysql/microsoft_mdm.go index a18d533e4d7..02e1341bb1d 100644 --- a/server/datastore/mysql/microsoft_mdm.go +++ b/server/datastore/mysql/microsoft_mdm.go @@ -2606,7 +2606,7 @@ INSERT INTO if len(labels) == 0 { profsWithoutLabel = append(profsWithoutLabel, profileUUID) } - if _, err := batchSetProfileLabelAssociationsDB(ctx, tx, labels, profsWithoutLabel, "windows"); err != nil { + if _, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, labels, profsWithoutLabel, "windows"); err != nil { return ctxerr.Wrap(ctx, err, "inserting windows profile label associations") } @@ -2618,7 +2618,7 @@ INSERT INTO FleetVariables: usesFleetVars, }, } - if _, err := batchSetProfileVariableAssociationsDB(ctx, tx, profilesVarsToUpsert, "windows"); err != nil { + if _, err := batchSetProfileVariableAssociationsDB(ctx, tx, ds.dialect, profilesVarsToUpsert, "windows"); err != nil { return ctxerr.Wrap(ctx, err, "inserting windows profile variable associations") } } diff --git a/server/datastore/mysql/nanomdm_storage.go b/server/datastore/mysql/nanomdm_storage.go index 6e8fb96c04e..d5b9290554f 100644 --- a/server/datastore/mysql/nanomdm_storage.go +++ b/server/datastore/mysql/nanomdm_storage.go @@ -54,9 +54,10 @@ func isConflict(err error) bool { type NanoMDMStorage struct { *nanomdm_mysql.MySQLStorage - db *sqlx.DB - logger *slog.Logger - ds fleet.Datastore + db *sqlx.DB + logger *slog.Logger + ds fleet.Datastore + dialect DialectHelper } // NewMDMAppleMDMStorage returns a MySQL nanomdm storage that uses the Datastore @@ -75,6 +76,7 @@ func (ds *Datastore) NewMDMAppleMDMStorage() (*NanoMDMStorage, error) { db: ds.primary, logger: ds.logger, ds: ds, + dialect: ds.dialect, }, nil } @@ -96,6 +98,7 @@ func (ds *Datastore) NewTestMDMAppleMDMStorage(asyncCap int, asyncInterval time. db: ds.primary, logger: ds.logger, ds: ds, + dialect: ds.dialect, }, nil } @@ -213,11 +216,11 @@ func (s *NanoMDMStorage) EnqueueDeviceLockCommand( fleet_platform ) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + s.dialect.OnDuplicateKey("host_id", ` wipe_ref = NULL, unlock_ref = NULL, unlock_pin = VALUES(unlock_pin), - lock_ref = VALUES(lock_ref)` + lock_ref = VALUES(lock_ref)`) if _, err := tx.ExecContext(ctx, stmt, host.ID, cmd.CommandUUID, pin, host.FleetPlatform()); err != nil { return ctxerr.Wrap(ctx, err, "modifying host_mdm_actions for DeviceLock") @@ -240,9 +243,9 @@ func (s *NanoMDMStorage) EnqueueDeviceUnlockCommand(ctx context.Context, host *f fleet_platform ) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + s.dialect.OnDuplicateKey("host_id", ` unlock_ref = VALUES(unlock_ref), - unlock_pin = NULL` + unlock_pin = NULL`) if _, err := tx.ExecContext(ctx, stmt, host.ID, cmd.CommandUUID, host.FleetPlatform()); err != nil { return ctxerr.Wrap(ctx, err, "modifying host_mdm_actions for DeviceUnlock") @@ -266,8 +269,7 @@ func (s *NanoMDMStorage) EnqueueDeviceWipeCommand(ctx context.Context, host *fle fleet_platform ) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE - wipe_ref = VALUES(wipe_ref)` + ` + s.dialect.OnDuplicateKey("host_id", "wipe_ref = VALUES(wipe_ref)") if _, err := tx.ExecContext(ctx, stmt, host.ID, cmd.CommandUUID, host.FleetPlatform()); err != nil { return ctxerr.Wrap(ctx, err, "modifying host_mdm_actions for DeviceWipe") diff --git a/server/datastore/mysql/operating_system_vulnerabilities.go b/server/datastore/mysql/operating_system_vulnerabilities.go index 832844da61f..c21823f9be7 100644 --- a/server/datastore/mysql/operating_system_vulnerabilities.go +++ b/server/datastore/mysql/operating_system_vulnerabilities.go @@ -57,12 +57,14 @@ func (ds *Datastore) ListVulnsByOsNameAndVersion(ctx context.Context, name, vers } // Query with CVSS metadata - baseCTE := ` + gcDistinctResolved := ds.dialect.GroupConcat("DISTINCT v.resolved_in_version", ",") + gcDistinctResolvedOsvv := ds.dialect.GroupConcat("DISTINCT osvv.resolved_in_version", ",") + baseCTE := fmt.Sprintf(` WITH all_vulns AS ( SELECT v.cve, MIN(v.created_at) created_at, - GROUP_CONCAT(DISTINCT v.resolved_in_version SEPARATOR ',') resolved_in_version + %s resolved_in_version FROM operating_system_vulnerabilities v JOIN operating_systems os ON os.id = v.operating_system_id AND os.name = ? AND os.version = ? @@ -73,14 +75,14 @@ func (ds *Datastore) ListVulnsByOsNameAndVersion(ctx context.Context, name, vers SELECT DISTINCT osvv.cve, MIN(osvv.created_at) created_at, - GROUP_CONCAT(DISTINCT osvv.resolved_in_version SEPARATOR ',') resolved_in_version + %s resolved_in_version FROM operating_system_version_vulnerabilities osvv JOIN operating_systems os ON os.os_version_id = osvv.os_version_id WHERE os.name = ? AND os.version = ? - ` + linuxTeamFilter + ` + `, gcDistinctResolved, gcDistinctResolvedOsvv) + linuxTeamFilter + ` GROUP BY osvv.cve ) ` @@ -294,11 +296,11 @@ func (ds *Datastore) InsertOSVulnerabilities(ctx context.Context, vulnerabilitie stmt := fmt.Sprintf(` INSERT INTO operating_system_vulnerabilities (operating_system_id, cve, source, resolved_in_version) VALUES %s - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("id", ` source = VALUES(source), resolved_in_version = VALUES(resolved_in_version), updated_at = NOW() - `, values) + `), values) var args []any for _, v := range batch { @@ -333,15 +335,27 @@ func (ds *Datastore) InsertOSVulnerability(ctx context.Context, v fleet.OSVulner source, resolved_in_version ) VALUES (?,?,?,?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("id", ` operating_system_id = VALUES(operating_system_id), source = VALUES(source), resolved_in_version = VALUES(resolved_in_version), updated_at = NOW() - ` + `) args = append(args, v.OSID, v.CVE, s, v.ResolvedInVersion) + if ds.dialect.ReturningID() != "" { + // PostgreSQL: use RETURNING id and xmax to distinguish insert from update. + // xmax = 0 means the row was freshly inserted (not updated). + var id int64 + var xmax uint32 + err := ds.writer(ctx).QueryRowContext(ctx, sqlStmt+" RETURNING id, xmax", args...).Scan(&id, &xmax) + if err != nil { + return false, ctxerr.Wrap(ctx, err, "insert operating system vulnerability") + } + return xmax == 0, nil + } + // MySQL path res, err := ds.writer(ctx).ExecContext(ctx, sqlStmt, args...) if err != nil { return false, ctxerr.Wrap(ctx, err, "insert operating system vulnerability") @@ -350,7 +364,11 @@ func (ds *Datastore) InsertOSVulnerability(ctx context.Context, v fleet.OSVulner // inserts affect one row, updates affect 0 or 2; we don't care which because timestamp may not change if we // recently inserted the vuln and changed nothing else; see insertOnDuplicateDidInsertOrUpdate for context affected, _ := res.RowsAffected() - lastID, _ := res.LastInsertId() + lastID, err := res.LastInsertId() + if err != nil { + // PG: no LastInsertId, use RowsAffected == 1 as insert indicator + return affected == 1, nil + } return lastID != 0 && affected == 1, nil } @@ -389,9 +407,11 @@ func (ds *Datastore) DeleteOutOfDateOSVulnerabilities(ctx context.Context, src f func (ds *Datastore) DeleteOrphanedOSVulnerabilities(ctx context.Context) error { if _, err := ds.writer(ctx).ExecContext(ctx, ` - DELETE osv FROM operating_system_vulnerabilities osv - LEFT JOIN host_operating_system hos ON hos.os_id = osv.operating_system_id - WHERE hos.host_id IS NULL + DELETE FROM operating_system_vulnerabilities + WHERE NOT EXISTS ( + SELECT 1 FROM host_operating_system hos + WHERE hos.os_id = operating_system_vulnerabilities.operating_system_id + ) `); err != nil { return ctxerr.Wrap(ctx, err, "deleting orphaned OS vulnerabilities") } @@ -603,12 +623,12 @@ func (ds *Datastore) refreshOSVersionVulnerabilities(ctx context.Context) error JOIN software_cve sc ON sc.software_id = khc.software_id WHERE khc.hosts_count > 0 GROUP BY khc.team_id, khc.os_version_id, sc.cve - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("id", ` source = VALUES(source), resolved_in_version = VALUES(resolved_in_version), created_at = VALUES(created_at), updated_at = VALUES(updated_at) - `, updatedAt) + `), updatedAt) if err != nil { return ctxerr.Wrap(ctx, err, "refresh per-team OS version vulnerabilities") } @@ -629,12 +649,12 @@ func (ds *Datastore) refreshOSVersionVulnerabilities(ctx context.Context) error JOIN software_cve sc ON sc.software_id = khc.software_id WHERE khc.hosts_count > 0 GROUP BY khc.os_version_id, sc.cve - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("id", ` source = VALUES(source), resolved_in_version = VALUES(resolved_in_version), created_at = VALUES(created_at), updated_at = VALUES(updated_at) - `, updatedAt) + `), updatedAt) if err != nil { return ctxerr.Wrap(ctx, err, "refresh all-teams OS version vulnerabilities") } diff --git a/server/datastore/mysql/operating_systems.go b/server/datastore/mysql/operating_systems.go index 2f9cbf30e46..f6e6ff96de5 100644 --- a/server/datastore/mysql/operating_systems.go +++ b/server/datastore/mysql/operating_systems.go @@ -56,7 +56,7 @@ func (ds *Datastore) UpdateHostOperatingSystem(ctx context.Context, hostID uint, if err != nil { return err } - return upsertHostOperatingSystemDB(ctx, tx, hostID, os.ID) + return upsertHostOperatingSystemDB(ctx, tx, ds.dialect, hostID, os.ID) }) } @@ -174,13 +174,13 @@ func isHostOperatingSystemUpdateNeeded(ctx context.Context, qc sqlx.QueryerConte // upsertHostOperatingSystemDB upserts the host operating system table // with the operating system id for the given host ID -func upsertHostOperatingSystemDB(ctx context.Context, tx sqlx.ExtContext, hostID uint, osID uint) error { +func upsertHostOperatingSystemDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, hostID uint, osID uint) error { // We do not use the `UPDATE` then `INSERT` pattern here because it causes a deadlock when multiple hosts are enrolled concurrently. // This method will rarely be called -- only when the host_operating_system needs to be updated. _, err := tx.ExecContext( ctx, `INSERT INTO host_operating_system (host_id, os_id) VALUES (?, ?) - ON DUPLICATE KEY UPDATE os_id = VALUES(os_id)`, hostID, osID, + `+dialect.OnDuplicateKey("host_id", "os_id = VALUES(os_id)"), hostID, osID, ) return err } diff --git a/server/datastore/mysql/packs.go b/server/datastore/mysql/packs.go index cb1cf9142e8..933a606bbe6 100644 --- a/server/datastore/mysql/packs.go +++ b/server/datastore/mysql/packs.go @@ -14,7 +14,7 @@ import ( func (ds *Datastore) ApplyPackSpecs(ctx context.Context, specs []*fleet.PackSpec) (err error) { err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { for _, spec := range specs { - if err := applyPackSpecDB(ctx, tx, spec); err != nil { + if err := applyPackSpecDB(ctx, tx, ds.dialect, spec); err != nil { return ctxerr.Wrapf(ctx, err, "applying pack '%s'", spec.Name) } } @@ -25,7 +25,7 @@ func (ds *Datastore) ApplyPackSpecs(ctx context.Context, specs []*fleet.PackSpec return err } -func applyPackSpecDB(ctx context.Context, tx sqlx.ExtContext, spec *fleet.PackSpec) error { +func applyPackSpecDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, spec *fleet.PackSpec) error { if spec.Name == "" { return ctxerr.New(ctx, "pack name must not be empty") } @@ -34,11 +34,11 @@ func applyPackSpecDB(ctx context.Context, tx sqlx.ExtContext, spec *fleet.PackSp query := ` INSERT INTO packs (name, description, platform, disabled) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + dialect.OnDuplicateKey("name", ` name = VALUES(name), description = VALUES(description), platform = VALUES(platform), - disabled = VALUES(disabled) + disabled = VALUES(disabled)`) + ` ` if _, err := tx.ExecContext(ctx, query, spec.Name, spec.Description, spec.Platform, spec.Disabled); err != nil { return ctxerr.Wrap(ctx, err, "insert/update pack") @@ -266,12 +266,11 @@ func (ds *Datastore) NewPack(ctx context.Context, pack *fleet.Pack, opts ...flee (name, description, platform, disabled) VALUES ( ?, ?, ?, ? ) ` - result, err := tx.ExecContext(ctx, query, pack.Name, pack.Description, pack.Platform, pack.Disabled) + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, query, pack.Name, pack.Description, pack.Platform, pack.Disabled) if err != nil { return ctxerr.Wrap(ctx, err, "insert pack") } - id, _ := result.LastInsertId() pack.ID = uint(id) //nolint:gosec // dismiss G115 if err := replacePackTargetsDB(ctx, tx, pack); err != nil { @@ -480,13 +479,8 @@ func listPacksForHost(ctx context.Context, db sqlx.QueryerContext, hid uint) ([] SELECT DISTINCT packs.* FROM ( ( SELECT p.* FROM packs p - JOIN pack_targets pt - JOIN label_membership lm - ON ( - p.id = pt.pack_id - AND pt.target_id = lm.label_id - AND pt.type = ? - ) + JOIN pack_targets pt ON p.id = pt.pack_id AND pt.type = ? + JOIN label_membership lm ON pt.target_id = lm.label_id WHERE lm.host_id = ? AND NOT p.disabled AND p.pack_type IS NULL ) UNION ALL diff --git a/server/datastore/mysql/password_reset.go b/server/datastore/mysql/password_reset.go index 4edd3f39514..fa3947a3388 100644 --- a/server/datastore/mysql/password_reset.go +++ b/server/datastore/mysql/password_reset.go @@ -17,14 +17,14 @@ func (ds *Datastore) NewPasswordResetRequest(ctx context.Context, req *fleet.Pas sqlStatement := ` INSERT INTO password_reset_requests ( user_id, token, expires_at) - VALUES (?,?, DATE_ADD(CURRENT_TIMESTAMP, INTERVAL ? MINUTE)) + VALUES (?,?, ?) ` - response, err := ds.writer(ctx).ExecContext(ctx, sqlStatement, req.UserID, req.Token, PasswordResetRequestDuration.Minutes()) + expiresAt := time.Now().Add(PasswordResetRequestDuration) + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), sqlStatement, req.UserID, req.Token, expiresAt) if err != nil { return nil, ctxerr.Wrap(ctx, err, "inserting password reset requests") } - id, _ := response.LastInsertId() req.ID = uint(id) //nolint:gosec // dismiss G115 return req, nil } diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index bb0295ee942..982736985ac 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -2,6 +2,7 @@ package mysql import ( "context" + "crypto/md5" "database/sql" "encoding/json" "errors" @@ -81,7 +82,7 @@ func (ds *Datastore) NewGlobalPolicy(ctx context.Context, authorID *uint, args f var newPolicy *fleet.Policy if err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { - p, err := newGlobalPolicy(ctx, tx, authorID, args) + p, err := newGlobalPolicy(ctx, tx, authorID, args, ds.dialect) if err != nil { return err } @@ -94,7 +95,7 @@ func (ds *Datastore) NewGlobalPolicy(ctx context.Context, authorID *uint, args f return newPolicy, nil } -func newGlobalPolicy(ctx context.Context, db sqlx.ExtContext, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) { +func newGlobalPolicy(ctx context.Context, db sqlx.ExtContext, authorID *uint, args fleet.PolicyPayload, dialect DialectHelper) (*fleet.Policy, error) { if args.SoftwareInstallerID != nil { return nil, ctxerr.Wrap(ctx, errSoftwareTitleIDOnGlobalPolicy, "create policy") } @@ -102,7 +103,7 @@ func newGlobalPolicy(ctx context.Context, db sqlx.ExtContext, authorID *uint, ar return nil, ctxerr.Wrap(ctx, errScriptIDOnGlobalPolicy, "create policy") } if args.QueryID != nil { - q, err := query(ctx, db, *args.QueryID) + q, err := query(ctx, db, *args.QueryID, dialect) if err != nil { return nil, ctxerr.Wrap(ctx, err, "fetching query from id") } @@ -112,12 +113,9 @@ func newGlobalPolicy(ctx context.Context, db sqlx.ExtContext, authorID *uint, ar } // We must normalize the name for full Unicode support (Unicode equivalence). nameUnicode := norm.NFC.String(args.Name) - res, err := db.ExecContext(ctx, - fmt.Sprintf( - `INSERT INTO policies (name, query, description, resolution, author_id, platforms, critical, checksum) VALUES (?, ?, ?, ?, ?, ?, ?, %s)`, - policiesChecksumComputedColumn(), - ), - nameUnicode, args.Query, args.Description, args.Resolution, authorID, args.Platform, args.Critical, + lastIdInt64, err := insertAndGetIDTx(ctx, db, dialect, + `INSERT INTO policies (name, query, description, resolution, author_id, platforms, critical, checksum) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + nameUnicode, args.Query, args.Description, args.Resolution, authorID, args.Platform, args.Critical, policyChecksum(nil, nameUnicode), ) switch { case err == nil: @@ -127,10 +125,6 @@ func newGlobalPolicy(ctx context.Context, db sqlx.ExtContext, authorID *uint, ar default: return nil, ctxerr.Wrap(ctx, err, "inserting new policy") } - lastIdInt64, err := res.LastInsertId() - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "getting last id after inserting policy") - } policyID := uint(lastIdInt64) //nolint:gosec // dismiss G115 dummyPolicy := &fleet.Policy{ @@ -280,6 +274,17 @@ func policiesChecksumComputedColumn() string { ) ` } +// policyChecksum computes the checksum for a policy in Go (portable across databases). +// The checksum is MD5(CONCAT_WS(\x00, COALESCE(team_id, ''), name)) as raw bytes. +func policyChecksum(teamID *uint, name string) []byte { + var teamStr string + if teamID != nil { + teamStr = fmt.Sprintf("%d", *teamID) + } + h := md5.Sum([]byte(teamStr + "\x00" + name)) + return h[:] +} + func (ds *Datastore) Policy(ctx context.Context, id uint) (*fleet.Policy, error) { return policyDB(ctx, ds.reader(ctx), id, nil) } @@ -341,7 +346,7 @@ func (ds *Datastore) PolicyLite(ctx context.Context, id uint) (*fleet.PolicyLite // Currently, SavePolicy does not allow updating the team of an existing policy. func (ds *Datastore) SavePolicy(ctx context.Context, p *fleet.Policy, shouldRemoveAllPolicyMemberships bool, removePolicyStats bool) error { if err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return savePolicy(ctx, tx, ds.logger, p, shouldRemoveAllPolicyMemberships, removePolicyStats) + return savePolicy(ctx, tx, ds.logger, p, shouldRemoveAllPolicyMemberships, removePolicyStats, ds.dialect) }); err != nil { return ctxerr.Wrap(ctx, err, "updating policy") } @@ -349,7 +354,7 @@ func (ds *Datastore) SavePolicy(ctx context.Context, p *fleet.Policy, shouldRemo return nil } -func savePolicy(ctx context.Context, db sqlx.ExtContext, logger *slog.Logger, p *fleet.Policy, shouldRemoveAllPolicyMemberships bool, removePolicyStats bool) error { +func savePolicy(ctx context.Context, db sqlx.ExtContext, logger *slog.Logger, p *fleet.Policy, shouldRemoveAllPolicyMemberships bool, removePolicyStats bool, dialect DialectHelper) error { if p.TeamID == nil && p.SoftwareInstallerID != nil { return ctxerr.Wrap(ctx, errSoftwareTitleIDOnGlobalPolicy, "save policy") } @@ -370,11 +375,11 @@ func savePolicy(ctx context.Context, db sqlx.ExtContext, logger *slog.Logger, p SET name = ?, query = ?, description = ?, resolution = ?, platforms = ?, critical = ?, calendar_events_enabled = ?, software_installer_id = ?, script_id = ?, vpp_apps_teams_id = ?, - conditional_access_enabled = ?, checksum = ` + policiesChecksumComputedColumn() + ` + conditional_access_enabled = ?, checksum = ? WHERE id = ? ` result, err := db.ExecContext( - ctx, updateStmt, p.Name, p.Query, p.Description, p.Resolution, p.Platform, p.Critical, p.CalendarEventsEnabled, p.SoftwareInstallerID, p.ScriptID, p.VPPAppsTeamsID, p.ConditionalAccessEnabled, p.ID, + ctx, updateStmt, p.Name, p.Query, p.Description, p.Resolution, p.Platform, p.Critical, p.CalendarEventsEnabled, p.SoftwareInstallerID, p.ScriptID, p.VPPAppsTeamsID, p.ConditionalAccessEnabled, policyChecksum(p.TeamID, p.Name), p.ID, ) if err != nil { return ctxerr.Wrap(ctx, err, "updating policy") @@ -397,7 +402,7 @@ func savePolicy(ctx context.Context, db sqlx.ExtContext, logger *slog.Logger, p } return cleanupPolicy( - ctx, db, db, p.ID, p.Platform, shouldRemoveAllPolicyMemberships, removePolicyStats, logger, + ctx, db, db, p.ID, p.Platform, shouldRemoveAllPolicyMemberships, removePolicyStats, logger, dialect, ) } @@ -492,14 +497,14 @@ func assertTeamMatches(ctx context.Context, db sqlx.QueryerContext, teamID uint, func cleanupPolicy( ctx context.Context, queryerContext sqlx.QueryerContext, extContext sqlx.ExtContext, policyID uint, policyPlatform string, shouldRemoveAllPolicyMemberships bool, - removePolicyStats bool, logger *slog.Logger, + removePolicyStats bool, logger *slog.Logger, dialect DialectHelper, ) error { var err error if shouldRemoveAllPolicyMemberships { - err = cleanupPolicyMembershipForPolicy(ctx, queryerContext, extContext, policyID) + err = cleanupPolicyMembershipForPolicy(ctx, queryerContext, extContext, dialect, policyID) } else { - err = cleanupPolicyMembershipOnPolicyUpdate(ctx, queryerContext, extContext, policyID, policyPlatform) + err = cleanupPolicyMembershipOnPolicyUpdate(ctx, queryerContext, extContext, policyID, policyPlatform, dialect) } if err != nil { return err @@ -649,9 +654,9 @@ func (ds *Datastore) RecordPolicyQueryExecutions(ctx context.Context, host *flee if len(results) > 0 { query := fmt.Sprintf( `INSERT INTO policy_membership (updated_at, policy_id, host_id, passes) - VALUES %s ON DUPLICATE KEY UPDATE updated_at=VALUES(updated_at), passes=VALUES(passes)`, + VALUES %s `, strings.Join(bindvars, ","), - ) + ) + ds.dialect.OnDuplicateKey("policy_id,host_id", "updated_at=VALUES(updated_at), passes=VALUES(passes)") if _, err := tx.ExecContext(ctx, query, vals...); err != nil { return ctxerr.Wrapf(ctx, err, "insert policy_membership (%v)", vals) } @@ -1009,7 +1014,7 @@ func (ds *Datastore) PolicyQueriesForHost(ctx context.Context, host *fleet.Host) // won't be receiving any policies targeted for specific platforms. ds.logger.ErrorContext(ctx, "unrecognized platform", "hostID", host.ID, "platform", host.Platform) } - const stmt = ` + stmt := fmt.Sprintf(` SELECT p.id, p.query FROM policies p WHERE @@ -1017,7 +1022,7 @@ func (ds *Datastore) PolicyQueriesForHost(ctx context.Context, host *fleet.Host) -- team_id == 0 are policies that apply to hosts in "No team" -- team_id > 0 are policies that apply to hosts in teams (team_id IS NULL OR team_id = COALESCE(?, 0)) AND - (platforms = '' OR FIND_IN_SET(?, platforms)) AND + (platforms = '' OR %s) AND ( -- Policy has no include labels NOT EXISTS ( @@ -1043,7 +1048,7 @@ func (ds *Datastore) PolicyQueriesForHost(ctx context.Context, host *fleet.Host) WHERE pl.policy_id = p.id AND pl.exclude = 1 ) -` +`, ds.dialect.FindInSet("?", "platforms")) var rows []struct { ID string `db:"id"` Query string `db:"query"` @@ -1086,7 +1091,7 @@ func (ds *Datastore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *u } if err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { - p, err := newTeamPolicy(ctx, tx, teamID, authorID, args) + p, err := newTeamPolicy(ctx, tx, teamID, authorID, args, ds.dialect) if err != nil { return err } @@ -1099,9 +1104,9 @@ func (ds *Datastore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *u return newPolicy, nil } -func newTeamPolicy(ctx context.Context, db sqlx.ExtContext, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) { +func newTeamPolicy(ctx context.Context, db sqlx.ExtContext, teamID uint, authorID *uint, args fleet.PolicyPayload, dialect DialectHelper) (*fleet.Policy, error) { if args.QueryID != nil { - q, err := query(ctx, db, *args.QueryID) + q, err := query(ctx, db, *args.QueryID, dialect) if err != nil { return nil, ctxerr.Wrap(ctx, err, "fetching query from id") } @@ -1128,19 +1133,16 @@ func newTeamPolicy(ctx context.Context, db sqlx.ExtContext, teamID uint, authorI return nil, ctxerr.Wrap(ctx, err, "create team policy") } - res, err := db.ExecContext(ctx, - fmt.Sprintf( - `INSERT INTO policies ( - name, query, description, team_id, resolution, author_id, - platforms, critical, calendar_events_enabled, software_installer_id, - script_id, vpp_apps_teams_id, conditional_access_enabled, checksum, - type, patch_software_title_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, %s, ?, ?)`, - policiesChecksumComputedColumn(), - ), + lastIdInt64, err := insertAndGetIDTx(ctx, db, dialect, + `INSERT INTO policies ( + name, query, description, team_id, resolution, author_id, + platforms, critical, calendar_events_enabled, software_installer_id, + script_id, vpp_apps_teams_id, conditional_access_enabled, checksum, + type, patch_software_title_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, nameUnicode, args.Query, args.Description, teamID, args.Resolution, authorID, args.Platform, args.Critical, args.CalendarEventsEnabled, args.SoftwareInstallerID, args.ScriptID, args.VPPAppsTeamsID, - args.ConditionalAccessEnabled, args.Type, args.PatchSoftwareTitleID, + args.ConditionalAccessEnabled, policyChecksum(&teamID, nameUnicode), args.Type, args.PatchSoftwareTitleID, ) switch { case err == nil: @@ -1154,10 +1156,6 @@ func newTeamPolicy(ctx context.Context, db sqlx.ExtContext, teamID uint, authorI default: return nil, ctxerr.Wrap(ctx, err, "inserting new policy") } - lastIdInt64, err := res.LastInsertId() - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "getting last id after inserting policy") - } policyID := uint(lastIdInt64) //nolint:gosec // dismiss G115 @@ -1418,8 +1416,7 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs // Reset on retry so we don't accumulate duplicate cleanup entries. pendingCleanups = pendingCleanups[:0] - query := fmt.Sprintf( - ` + query := ` INSERT INTO policies ( name, query, @@ -1437,9 +1434,8 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs checksum, type, patch_software_title_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, %s, ?, ?) - ON DUPLICATE KEY UPDATE - query = VALUES(query), + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ` + ds.dialect.OnDuplicateKey("checksum", `query = VALUES(query), description = VALUES(description), author_id = VALUES(author_id), resolution = VALUES(resolution), @@ -1451,9 +1447,7 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs script_id = VALUES(script_id), conditional_access_enabled = VALUES(conditional_access_enabled), type = VALUES(type), - patch_software_title_id = VALUES(patch_software_title_id) - `, policiesChecksumComputedColumn(), - ) + patch_software_title_id = VALUES(patch_software_title_id)`) for teamID, teamPolicySpecs := range teamIDToPolicies { for _, spec := range teamPolicySpecs { var softwareInstallerID *uint @@ -1500,7 +1494,7 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs query, spec.Name, spec.Query, spec.Description, authorID, spec.Resolution, teamID, spec.Platform, spec.Critical, spec.CalendarEventsEnabled, softwareInstallerID, vppAppsTeamsID, scriptID, spec.ConditionalAccessEnabled, - spec.Type, fmaTitleID, + policyChecksum(teamID, norm.NFC.String(spec.Name)), spec.Type, fmaTitleID, ) if err != nil { return ctxerr.Wrap(ctx, err, "exec ApplyPolicySpecs insert") @@ -1636,6 +1630,7 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs args.shouldRemoveAllPolicyMemberships, args.removePolicyStats, ds.logger, + ds.dialect, ); err != nil { return err } @@ -1668,10 +1663,10 @@ func (ds *Datastore) AsyncBatchInsertPolicyMembership(ctx context.Context, batch // INSERT IGNORE, to avoid failing if policy / host does not exist (as this // runs asynchronously, they could get deleted in between the data being // received and being upserted). - sql := `INSERT IGNORE INTO policy_membership (policy_id, host_id, passes) VALUES ` + sql := ds.dialect.InsertIgnoreInto() + ` policy_membership (policy_id, host_id, passes) VALUES ` sql += strings.Repeat(`(?, ?, ?),`, len(batch)) sql = strings.TrimSuffix(sql, ",") - sql += ` ON DUPLICATE KEY UPDATE updated_at = VALUES(updated_at), passes = VALUES(passes)` + sql += ` ` + ds.dialect.OnDuplicateKey("policy_id,host_id", "updated_at = VALUES(updated_at), passes = VALUES(passes)") vals := make([]interface{}, 0, len(batch)*3) hostIDs := make([]uint, 0, len(batch)) @@ -1757,19 +1752,19 @@ func (ds *Datastore) AsyncBatchUpdatePolicyTimestamp(ctx context.Context, ids [] }) } -func deleteAllPolicyMemberships(ctx context.Context, tx sqlx.ExtContext, hostID uint) error { +func deleteAllPolicyMemberships(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, hostID uint) error { query := `DELETE FROM policy_membership WHERE host_id = ?` if _, err := tx.ExecContext(ctx, query, hostID); err != nil { return ctxerr.Wrap(ctx, err, "exec delete policies") } // Use the single host method for better performance and no unnecessary locking - if err := updateHostIssuesFailingPoliciesForSingleHost(ctx, tx, hostID); err != nil { + if err := updateHostIssuesFailingPoliciesForSingleHost(ctx, tx, dialect, hostID); err != nil { return err } return nil } -func cleanupPolicyMembershipOnTeamChange(ctx context.Context, tx sqlx.ExtContext, hostIDs []uint) error { +func cleanupPolicyMembershipOnTeamChange(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, hostIDs []uint) error { // hosts can only be in one team, so if there's a policy that has a team id and a result from one of our hosts // it can only be from the previous team they are being transferred from query, args, err := sqlx.In(`DELETE FROM policy_membership @@ -1782,7 +1777,7 @@ func cleanupPolicyMembershipOnTeamChange(ctx context.Context, tx sqlx.ExtContext } // This method is currently called for a batch of hosts. Performance should be monitored. If performance becomes a concern, // we can reduce batch size or move this method outside the transaction. - if err = updateHostIssuesFailingPolicies(ctx, tx, hostIDs); err != nil { + if err = updateHostIssuesFailingPolicies(ctx, tx, dialect, hostIDs); err != nil { return err } return nil @@ -1832,7 +1827,7 @@ func cleanupConditionalAccessOnTeamChange(ctx context.Context, tx sqlx.ExtContex } func cleanupPolicyMembershipOnPolicyUpdate( - ctx context.Context, queryerContext sqlx.QueryerContext, db sqlx.ExecerContext, policyID uint, platforms string, + ctx context.Context, queryerContext sqlx.QueryerContext, db sqlx.ExecerContext, policyID uint, platforms string, dialect DialectHelper, ) error { // Clean up hosts that don't match the platform criteria. // Page through rows using the (policy_id, host_id) PK as a cursor so each SELECT+DELETE @@ -1847,14 +1842,14 @@ func cleanupPolicyMembershipOnPolicyUpdate( var afterHostID uint for { var batchHostIDs []uint - err := sqlx.SelectContext(ctx, queryerContext, &batchHostIDs, ` + err := sqlx.SelectContext(ctx, queryerContext, &batchHostIDs, fmt.Sprintf(` SELECT pm.host_id FROM policy_membership pm INNER JOIN hosts h ON pm.host_id = h.id - WHERE pm.policy_id = ? AND FIND_IN_SET(h.platform, ?) = 0 + WHERE pm.policy_id = ? AND %s = 0 AND pm.host_id > ? ORDER BY pm.host_id ASC - LIMIT ?`, policyID, expandedPlatformsStr, afterHostID, policyMembershipDeleteBatchSize) + LIMIT ?`, dialect.FindInSet("h.platform", "?")), policyID, expandedPlatformsStr, afterHostID, policyMembershipDeleteBatchSize) if err != nil { return ctxerr.Wrap(ctx, err, "select batch of hosts to cleanup policy membership for platform") } @@ -1872,16 +1867,15 @@ func cleanupPolicyMembershipOnPolicyUpdate( if _, err = db.ExecContext(ctx, batchStmt, args...); err != nil { return ctxerr.Wrap(ctx, err, "batch cleanup policy membership for platform") } - if err := updateHostIssuesFailingPolicies(ctx, db, batchHostIDs); err != nil { + if err := updateHostIssuesFailingPolicies(ctx, db, dialect, batchHostIDs); err != nil { return err } afterHostID = batchHostIDs[len(batchHostIDs)-1] } // Clean up orphaned memberships (host_id refs to deleted hosts, not covered by INNER JOIN above) if _, err := db.ExecContext(ctx, ` - DELETE pm FROM policy_membership pm - LEFT JOIN hosts h ON pm.host_id = h.id - WHERE pm.policy_id = ? AND h.id IS NULL`, policyID); err != nil { + DELETE FROM policy_membership + WHERE policy_id = ? AND NOT EXISTS (SELECT 1 FROM hosts WHERE hosts.id = policy_membership.host_id)`, policyID); err != nil { return ctxerr.Wrap(ctx, err, "cleanup orphaned policy membership for platform") } } @@ -1935,7 +1929,7 @@ func cleanupPolicyMembershipOnPolicyUpdate( if _, err = db.ExecContext(ctx, batchStmt, args...); err != nil { return ctxerr.Wrap(ctx, err, "batch cleanup policy membership for labels") } - if err := updateHostIssuesFailingPolicies(ctx, db, batchHostIDs); err != nil { + if err := updateHostIssuesFailingPolicies(ctx, db, dialect, batchHostIDs); err != nil { return err } afterLabelHostID = batchHostIDs[len(batchHostIDs)-1] @@ -1950,6 +1944,7 @@ func cleanupPolicyMembershipForPolicy( ctx context.Context, queryerContext sqlx.QueryerContext, exec sqlx.ExecerContext, + dialect DialectHelper, policyID uint, ) error { // Page through policy_membership using (policy_id, host_id) as a cursor. Selecting and deleting one @@ -1982,16 +1977,15 @@ func cleanupPolicyMembershipForPolicy( if _, err = exec.ExecContext(ctx, batchStmt, args...); err != nil { return ctxerr.Wrap(ctx, err, "batch cleanup policy membership") } - if err := updateHostIssuesFailingPolicies(ctx, exec, batchHostIDs); err != nil { + if err := updateHostIssuesFailingPolicies(ctx, exec, dialect, batchHostIDs); err != nil { return err } afterHostID = batchHostIDs[len(batchHostIDs)-1] } // Clean up orphaned memberships (host_id refs to deleted hosts, not covered by INNER JOIN above) if _, err := exec.ExecContext(ctx, ` - DELETE pm FROM policy_membership pm - LEFT JOIN hosts h ON pm.host_id = h.id - WHERE pm.policy_id = ? AND h.id IS NULL`, policyID); err != nil { + DELETE FROM policy_membership + WHERE policy_id = ? AND NOT EXISTS (SELECT 1 FROM hosts WHERE hosts.id = policy_membership.host_id)`, policyID); err != nil { return ctxerr.Wrap(ctx, err, "cleanup orphaned policy membership") } @@ -2015,17 +2009,17 @@ func (ds *Datastore) CleanupPolicyMembership(ctx context.Context, now time.Time) FROM policies p WHERE - p.updated_at >= DATE_SUB(?, INTERVAL ? SECOND) AND + p.updated_at >= ? AND p.created_at < p.updated_at` ) var pols []*fleet.Policy - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &pols, updatedPoliciesStmt, now, int(recentlyUpdatedPoliciesInterval.Seconds())); err != nil { + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &pols, updatedPoliciesStmt, now.Add(-recentlyUpdatedPoliciesInterval)); err != nil { return ctxerr.Wrap(ctx, err, "select recently updated policies") } for _, pol := range pols { - if err := cleanupPolicyMembershipOnPolicyUpdate(ctx, ds.reader(ctx), ds.writer(ctx), pol.ID, pol.Platform); err != nil { + if err := cleanupPolicyMembershipOnPolicyUpdate(ctx, ds.reader(ctx), ds.writer(ctx), pol.ID, pol.Platform, ds.dialect); err != nil { return ctxerr.Wrapf(ctx, err, "delete outdated hosts membership for policy: %d; platforms: %v", pol.ID, pol.Platform) } } @@ -2039,7 +2033,7 @@ func (ds *Datastore) CleanupPolicyMembership(ctx context.Context, now time.Time) return ctxerr.Wrap(ctx, err, "select policies needing full membership cleanup") } for _, polID := range fullCleanupPolIDs { - if err := cleanupPolicyMembershipForPolicy(ctx, ds.reader(ctx), ds.writer(ctx), polID); err != nil { + if err := cleanupPolicyMembershipForPolicy(ctx, ds.reader(ctx), ds.writer(ctx), ds.dialect, polID); err != nil { return ctxerr.Wrapf(ctx, err, "full membership cleanup for policy %d", polID) } if _, err := ds.writer(ctx).ExecContext(ctx, @@ -2064,7 +2058,7 @@ type PolicyViolationDays struct { func (ds *Datastore) IncrementPolicyViolationDays(ctx context.Context) error { return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return incrementViolationDaysDB(ctx, tx) + return incrementViolationDaysDB(ctx, tx, ds.dialect) }) } @@ -2094,8 +2088,8 @@ func (ds *Datastore) IncreasePolicyAutomationIteration(ctx context.Context, poli return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { _, err := tx.ExecContext(ctx, ` INSERT INTO policy_automation_iterations (policy_id, iteration) VALUES (?,1) - ON DUPLICATE KEY UPDATE iteration = iteration + 1; - `, policyID) + `+ds.dialect.OnDuplicateKey("policy_id", "iteration = policy_automation_iterations.iteration + 1"), + policyID) return err }) } @@ -2137,7 +2131,7 @@ func (ds *Datastore) OutdatedAutomationBatch(ctx context.Context) ([]fleet.Polic return nil } query := ` - UPDATE policy_membership pm SET pm.automation_iteration = ( + UPDATE policy_membership pm SET automation_iteration = ( SELECT ai.iteration FROM policy_automation_iterations ai WHERE pm.policy_id = ai.policy_id @@ -2155,7 +2149,7 @@ func (ds *Datastore) OutdatedAutomationBatch(ctx context.Context) ([]fleet.Polic return failures, nil } -func incrementViolationDaysDB(ctx context.Context, tx sqlx.ExtContext) error { +func incrementViolationDaysDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper) error { const ( statsID = 0 globalStats = true @@ -2211,7 +2205,7 @@ func incrementViolationDaysDB(ctx context.Context, tx sqlx.ExtContext) error { // `policy_membership` var newCounts PolicyViolationDays if err := sqlx.GetContext(ctx, tx, &newCounts, ` - SELECT (select count(*) from policy_membership where passes=0) as failing_host_count, + SELECT (select count(*) from policy_membership where passes = false) as failing_host_count, (select count(*) from policy_membership) as total_host_count`, ); err != nil { return ctxerr.Wrap(ctx, err, "count policy violation days") @@ -2228,8 +2222,7 @@ func incrementViolationDaysDB(ctx context.Context, tx sqlx.ExtContext) error { INSERT INTO aggregated_stats (id, global_stats, type, json_value) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE - json_value = VALUES(json_value)` + ` + dialect.OnDuplicateKey("id,type,global_stats", "json_value = VALUES(json_value), updated_at = NOW()") if _, err := tx.ExecContext(ctx, upsertStmt, statsID, globalStats, statsType, statsJSON); err != nil { return ctxerr.Wrap(ctx, err, "update policy violation days aggregated stats") } @@ -2239,11 +2232,11 @@ func incrementViolationDaysDB(ctx context.Context, tx sqlx.ExtContext) error { func (ds *Datastore) InitializePolicyViolationDays(ctx context.Context) error { return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return initializePolicyViolationDaysDB(ctx, tx) + return initializePolicyViolationDaysDB(ctx, tx, ds.dialect) }) } -func initializePolicyViolationDaysDB(ctx context.Context, tx sqlx.ExtContext) error { +func initializePolicyViolationDaysDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper) error { const ( statsID = 0 globalStats = true @@ -2259,9 +2252,8 @@ func initializePolicyViolationDaysDB(ctx context.Context, tx sqlx.ExtContext) er INSERT INTO aggregated_stats (id, global_stats, type, json_value) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE - json_value = VALUES(json_value), - created_at = CURRENT_TIMESTAMP` + ` + dialect.OnDuplicateKey("id,type,global_stats", `json_value = VALUES(json_value), + created_at = CURRENT_TIMESTAMP`) if _, err := tx.ExecContext(ctx, stmt, statsID, globalStats, statsType, statsJSON); err != nil { return ctxerr.Wrap(ctx, err, "initialize policy violation days aggregated stats") } @@ -2402,10 +2394,9 @@ func (ds *Datastore) UpdateHostPolicyCounts(ctx context.Context) error { insertStmt := `INSERT INTO policy_stats (policy_id, inherited_team_id, passing_host_count, failing_host_count) VALUES (:policy_id, :inherited_team_id, :passing_host_count, :failing_host_count) - ON DUPLICATE KEY UPDATE - updated_at = NOW(), + ` + ds.dialect.OnDuplicateKey("policy_id,inherited_team_id_char", `updated_at = NOW(), passing_host_count = VALUES(passing_host_count), - failing_host_count = VALUES(failing_host_count)` + failing_host_count = VALUES(failing_host_count)`) _, err = sqlx.NamedExecContext(ctx, db, insertStmt, policyStats) if err != nil { // INSERT may fail due to rare race conditions. We log and proceed. @@ -2429,11 +2420,9 @@ func (ds *Datastore) UpdateHostPolicyCounts(ctx context.Context) error { FROM policies p LEFT JOIN policy_membership pm ON p.id = pm.policy_id GROUP BY p.id - ON DUPLICATE KEY UPDATE - updated_at = NOW(), + `+ds.dialect.OnDuplicateKey("policy_id,inherited_team_id_char", `updated_at = NOW(), passing_host_count = VALUES(passing_host_count), - failing_host_count = VALUES(failing_host_count); - `) + failing_host_count = VALUES(failing_host_count)`)) if err != nil { return ctxerr.Wrap(ctx, err, "update host policy counts for global and team policies") } @@ -2517,7 +2506,7 @@ func (ds *Datastore) GetTeamHostsPolicyMemberships( policyIDs []uint, hostID *uint, ) ([]fleet.HostPolicyMembershipData, error) { - query := ` + query := fmt.Sprintf(` SELECT COALESCE(sh.email, '') AS email, COALESCE(pm.passing, 1) AS passing, @@ -2527,11 +2516,11 @@ func (ds *Datastore) GetTeamHostsPolicyMemberships( h.hardware_serial AS host_hardware_serial FROM hosts h LEFT JOIN ( - SELECT host_id, 0 AS passing, GROUP_CONCAT(policy_id) AS failing_policy_ids + SELECT host_id, 0 AS passing, %s AS failing_policy_ids FROM policy_membership - WHERE policy_id IN (?) AND passes = 0 + WHERE policy_id IN (?) AND passes = false GROUP BY host_id - ) pm ON h.id = pm.host_id + ) pm ON h.id = pm.host_id`, ds.dialect.GroupConcat("policy_id", ",")) + ` LEFT JOIN ( SELECT host_id, email FROM ( @@ -2556,7 +2545,7 @@ func (ds *Datastore) GetTeamHostsPolicyMemberships( ) sh ON h.id = sh.host_id LEFT JOIN host_display_names hdn ON h.id = hdn.host_id LEFT JOIN host_calendar_events hce ON h.id = hce.host_id - WHERE h.team_id = ? AND ((pm.passing IS NOT NULL AND NOT pm.passing) OR (COALESCE(pm.passing, 1) AND hce.host_id IS NOT NULL)) + WHERE h.team_id = ? AND ((pm.passing IS NOT NULL AND pm.passing = 0) OR (COALESCE(pm.passing, 1) = 1 AND hce.host_id IS NOT NULL)) ` query, args, err := sqlx.In(query, diff --git a/server/datastore/mysql/queries.go b/server/datastore/mysql/queries.go index b50be523914..105667cd742 100644 --- a/server/datastore/mysql/queries.go +++ b/server/datastore/mysql/queries.go @@ -65,7 +65,7 @@ func (ds *Datastore) applyQueriesInTx( } } - const upsertQueriesSQL = ` + upsertQueriesSQL := ` INSERT INTO queries ( name, description, @@ -82,8 +82,7 @@ func (ds *Datastore) applyQueriesInTx( logging_type, discard_data ) VALUES %s - ON DUPLICATE KEY UPDATE - name = VALUES(name), + ` + ds.dialect.OnDuplicateKey("id", `name = VALUES(name), description = VALUES(description), query = VALUES(query), author_id = VALUES(author_id), @@ -96,7 +95,7 @@ func (ds *Datastore) applyQueriesInTx( schedule_interval = VALUES(schedule_interval), automations_enabled = VALUES(automations_enabled), logging_type = VALUES(logging_type), - discard_data = VALUES(discard_data)` + discard_data = VALUES(discard_data)`) // 'queries' are uniquely identified by {name, team_id} unqKeyGen := func(name string, teamID *uint) string { @@ -279,8 +278,9 @@ func (ds *Datastore) NewQuery( ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) ` - result, err := ds.writer(ctx).ExecContext( + id, err := ds.insertAndGetID( ctx, + ds.writer(ctx), queryStatement, query.Name, query.Description, @@ -300,13 +300,12 @@ func (ds *Datastore) NewQuery( query.UpdatedAt, ) - if err != nil && IsDuplicate(err) { + if err != nil && ds.dialect.IsDuplicate(err) { return nil, ctxerr.Wrap(ctx, alreadyExists("Query", query.Name)) } else if err != nil { return nil, ctxerr.Wrap(ctx, err, "creating new Query") } - id, _ := result.LastInsertId() query.ID = uint(id) //nolint:gosec // dismiss G115 query.Packs = []fleet.Pack{} @@ -523,7 +522,7 @@ func (ds *Datastore) DeleteQuery(ctx context.Context, teamID *uint, name string) deleteStmt := "DELETE FROM queries WHERE id = ?" result, err := ds.writer(ctx).ExecContext(ctx, deleteStmt, queryID) if err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { return ctxerr.Wrap(ctx, foreignKey("queries", name)) } return ctxerr.Wrap(ctx, err, "delete queries") @@ -598,11 +597,11 @@ func (ds *Datastore) deleteQueryStats(ctx context.Context, queryIDs []uint) { // Query returns a single Query identified by id, if such exists. func (ds *Datastore) Query(ctx context.Context, id uint) (*fleet.Query, error) { - return query(ctx, ds.reader(ctx), id) + return query(ctx, ds.reader(ctx), id, ds.dialect) } -func query(ctx context.Context, db sqlx.QueryerContext, id uint) (*fleet.Query, error) { - sqlQuery := ` +func query(ctx context.Context, db sqlx.QueryerContext, id uint, dialect DialectHelper) (*fleet.Query, error) { + sqlQuery := fmt.Sprintf(` SELECT q.id, q.team_id, @@ -623,18 +622,24 @@ func query(ctx context.Context, db sqlx.QueryerContext, id uint) (*fleet.Query, q.discard_data, COALESCE(NULLIF(u.name, ''), u.email, '') AS author_name, COALESCE(u.email, '') AS author_email, - JSON_EXTRACT(json_value, '$.user_time_p50') as user_time_p50, - JSON_EXTRACT(json_value, '$.user_time_p95') as user_time_p95, - JSON_EXTRACT(json_value, '$.system_time_p50') as system_time_p50, - JSON_EXTRACT(json_value, '$.system_time_p95') as system_time_p95, - JSON_EXTRACT(json_value, '$.total_executions') as total_executions + %s as user_time_p50, + %s as user_time_p95, + %s as system_time_p50, + %s as system_time_p95, + %s as total_executions FROM queries q LEFT JOIN users u ON q.author_id = u.id LEFT JOIN aggregated_stats ag ON (ag.id = q.id AND ag.global_stats = ? AND ag.type = ?) WHERE q.id = ? - ` + `, + dialect.JSONExtract("json_value", "$.user_time_p50"), + dialect.JSONExtract("json_value", "$.user_time_p95"), + dialect.JSONExtract("json_value", "$.system_time_p50"), + dialect.JSONExtract("json_value", "$.system_time_p95"), + dialect.JSONExtract("json_value", "$.total_executions"), + ) query := &fleet.Query{} if err := sqlx.GetContext(ctx, db, query, sqlQuery, false, fleet.AggregatedStatsTypeScheduledQuery, id); err != nil { if err == sql.ErrNoRows { @@ -658,7 +663,7 @@ func query(ctx context.Context, db sqlx.QueryerContext, id uint) (*fleet.Query, // determined by passed in fleet.ListOptions, count of total queries returned without limits, and // pagination metadata func (ds *Datastore) ListQueries(ctx context.Context, opt fleet.ListQueryOptions) (queries []*fleet.Query, total int, inherited int, metadata *fleet.PaginationMetadata, err error) { - getQueriesStmt := ` + getQueriesStmt := fmt.Sprintf(` SELECT q.id, q.team_id, @@ -678,15 +683,21 @@ func (ds *Datastore) ListQueries(ctx context.Context, opt fleet.ListQueryOptions q.updated_at, COALESCE(u.name, '') AS author_name, COALESCE(u.email, '') AS author_email, - JSON_EXTRACT(json_value, '$.user_time_p50') as user_time_p50, - JSON_EXTRACT(json_value, '$.user_time_p95') as user_time_p95, - JSON_EXTRACT(json_value, '$.system_time_p50') as system_time_p50, - JSON_EXTRACT(json_value, '$.system_time_p95') as system_time_p95, - JSON_EXTRACT(json_value, '$.total_executions') as total_executions + %s as user_time_p50, + %s as user_time_p95, + %s as system_time_p50, + %s as system_time_p95, + %s as total_executions FROM queries q LEFT JOIN users u ON (q.author_id = u.id) LEFT JOIN aggregated_stats ag ON (ag.id = q.id AND ag.global_stats = ? AND ag.type = ?) - ` + `, + ds.dialect.JSONExtract("json_value", "$.user_time_p50"), + ds.dialect.JSONExtract("json_value", "$.user_time_p95"), + ds.dialect.JSONExtract("json_value", "$.system_time_p50"), + ds.dialect.JSONExtract("json_value", "$.system_time_p95"), + ds.dialect.JSONExtract("json_value", "$.total_executions"), + ) args := []interface{}{false, fleet.AggregatedStatsTypeScheduledQuery} whereClauses := "WHERE saved = true" @@ -1001,7 +1012,7 @@ func (ds *Datastore) UpdateLiveQueryStats(ctx context.Context, queryID uint, sta // Bulk insert/update const valueStr = "(?,?,?,?,?,?,?,?,?,?,?,?)," - stmt := "REPLACE INTO scheduled_query_stats (scheduled_query_id, host_id, query_type, executions, average_memory, system_time, user_time, wall_time, output_size, denylisted, schedule_interval, last_executed) VALUES " + + stmt := ds.dialect.ReplaceInto() + " scheduled_query_stats (scheduled_query_id, host_id, query_type, executions, average_memory, system_time, user_time, wall_time, output_size, denylisted, schedule_interval, last_executed) VALUES " + strings.Repeat(valueStr, len(stats)) stmt = strings.TrimSuffix(stmt, ",") diff --git a/server/datastore/mysql/query_results.go b/server/datastore/mysql/query_results.go index 0ad734ad1f3..edbef205fc7 100644 --- a/server/datastore/mysql/query_results.go +++ b/server/datastore/mysql/query_results.go @@ -54,9 +54,9 @@ func (ds *Datastore) OverwriteQueryResultRows(ctx context.Context, rows []*fleet } //nolint:gosec // SQL query is constructed using constant strings - insertStmt := ` - INSERT IGNORE INTO query_results (query_id, host_id, last_fetched, data) VALUES - ` + strings.Join(valueStrings, ",") + insertStmt := ds.dialect.InsertIgnoreInto() + ` + query_results (query_id, host_id, last_fetched, data) VALUES + ` + strings.Join(valueStrings, ",") + ds.dialect.OnConflictDoNothing("query_id,host_id") result, err = tx.ExecContext(ctx, insertStmt, valueArgs...) if err != nil { diff --git a/server/datastore/mysql/scheduled_queries.go b/server/datastore/mysql/scheduled_queries.go index 3c001b1634d..4f334c9e386 100644 --- a/server/datastore/mysql/scheduled_queries.go +++ b/server/datastore/mysql/scheduled_queries.go @@ -15,7 +15,7 @@ import ( // ListScheduledQueriesInPackWithStats loads a pack's scheduled queries and its aggregated stats. func (ds *Datastore) ListScheduledQueriesInPackWithStats(ctx context.Context, id uint, opts fleet.ListOptions) ([]*fleet.ScheduledQuery, error) { - query := ` + query := fmt.Sprintf(` SELECT sq.id, sq.pack_id, @@ -31,16 +31,22 @@ func (ds *Datastore) ListScheduledQueriesInPackWithStats(ctx context.Context, id sq.denylist, q.query, q.id AS query_id, - JSON_EXTRACT(ag.json_value, '$.user_time_p50') as user_time_p50, - JSON_EXTRACT(ag.json_value, '$.user_time_p95') as user_time_p95, - JSON_EXTRACT(ag.json_value, '$.system_time_p50') as system_time_p50, - JSON_EXTRACT(ag.json_value, '$.system_time_p95') as system_time_p95, - JSON_EXTRACT(ag.json_value, '$.total_executions') as total_executions + %s as user_time_p50, + %s as user_time_p95, + %s as system_time_p50, + %s as system_time_p95, + %s as total_executions FROM scheduled_queries sq JOIN (SELECT * FROM queries WHERE team_id IS NULL) q ON (sq.query_name = q.name) LEFT JOIN aggregated_stats ag ON (ag.id = sq.id AND ag.global_stats = ? AND ag.type = ?) WHERE sq.pack_id = ? - ` + `, + ds.dialect.JSONExtract("ag.json_value", "$.user_time_p50"), + ds.dialect.JSONExtract("ag.json_value", "$.user_time_p95"), + ds.dialect.JSONExtract("ag.json_value", "$.system_time_p50"), + ds.dialect.JSONExtract("ag.json_value", "$.system_time_p95"), + ds.dialect.JSONExtract("ag.json_value", "$.total_executions"), + ) params := []interface{}{false, fleet.AggregatedStatsTypeScheduledQuery, id} query, params = appendListOptionsWithCursorToSQL(query, params, &opts) results := []*fleet.ScheduledQuery{} @@ -83,10 +89,10 @@ func (ds *Datastore) ListScheduledQueriesInPack(ctx context.Context, id uint) (f } func (ds *Datastore) NewScheduledQuery(ctx context.Context, sq *fleet.ScheduledQuery, opts ...fleet.OptionalArg) (*fleet.ScheduledQuery, error) { - return insertScheduledQueryDB(ctx, ds.writer(ctx), sq) + return insertScheduledQueryDB(ctx, ds.writer(ctx), ds.dialect, sq) } -func insertScheduledQueryDB(ctx context.Context, q sqlx.ExtContext, sq *fleet.ScheduledQuery) (*fleet.ScheduledQuery, error) { +func insertScheduledQueryDB(ctx context.Context, q sqlx.ExtContext, dialect DialectHelper, sq *fleet.ScheduledQuery) (*fleet.ScheduledQuery, error) { // This query looks up the query name using the ID (for backwards // compatibility with the UI) query := ` @@ -97,7 +103,7 @@ func insertScheduledQueryDB(ctx context.Context, q sqlx.ExtContext, sq *fleet.Sc pack_id, snapshot, removed, - ` + "`interval`" + `, + "interval", platform, version, shard, @@ -107,12 +113,11 @@ func insertScheduledQueryDB(ctx context.Context, q sqlx.ExtContext, sq *fleet.Sc FROM queries WHERE id = ? ` - result, err := q.ExecContext(ctx, query, sq.QueryID, sq.Name, sq.PackID, sq.Snapshot, sq.Removed, sq.Interval, sq.Platform, sq.Version, sq.Shard, sq.Denylist, sq.QueryID) + id, err := insertAndGetIDTx(ctx, q, dialect, query, sq.QueryID, sq.Name, sq.PackID, sq.Snapshot, sq.Removed, sq.Interval, sq.Platform, sq.Version, sq.Shard, sq.Denylist, sq.QueryID) if err != nil { return nil, ctxerr.Wrap(ctx, err, "insert scheduled query") } - id, _ := result.LastInsertId() sq.ID = uint(id) //nolint:gosec // dismiss G115 query = `SELECT query, name FROM queries WHERE id = ? LIMIT 1` @@ -145,7 +150,7 @@ func (ds *Datastore) SaveScheduledQuery(ctx context.Context, sq *fleet.Scheduled func saveScheduledQueryDB(ctx context.Context, exec sqlx.ExecerContext, sq *fleet.ScheduledQuery) (*fleet.ScheduledQuery, error) { query := ` UPDATE scheduled_queries - SET pack_id = ?, query_id = ?, ` + "`interval`" + ` = ?, snapshot = ?, removed = ?, platform = ?, version = ?, shard = ?, denylist = ? + SET pack_id = ?, query_id = ?, "interval" = ?, snapshot = ?, removed = ?, platform = ?, version = ?, shard = ?, denylist = ? WHERE id = ? ` result, err := exec.ExecContext(ctx, query, sq.PackID, sq.QueryID, sq.Interval, sq.Snapshot, sq.Removed, sq.Platform, sq.Version, sq.Shard, sq.Denylist, sq.ID) @@ -276,8 +281,8 @@ func (ds *Datastore) AsyncBatchSaveHostsScheduledQueryStats(ctx context.Context, // in SaveHostPackStats (in hosts.go) - that is, the behaviour per host must // be the same. - stmt := ` - INSERT IGNORE INTO scheduled_query_stats ( + stmt := ds.dialect.InsertIgnoreInto() + ` + scheduled_query_stats ( scheduled_query_id, host_id, average_memory, @@ -290,7 +295,7 @@ func (ds *Datastore) AsyncBatchSaveHostsScheduledQueryStats(ctx context.Context, user_time, wall_time ) - VALUES %s ON DUPLICATE KEY UPDATE + VALUES %s ` + ds.dialect.OnDuplicateKey("scheduled_query_id,host_id", ` scheduled_query_id = VALUES(scheduled_query_id), host_id = VALUES(host_id), average_memory = VALUES(average_memory), @@ -301,7 +306,7 @@ func (ds *Datastore) AsyncBatchSaveHostsScheduledQueryStats(ctx context.Context, output_size = VALUES(output_size), system_time = VALUES(system_time), user_time = VALUES(user_time), - wall_time = VALUES(wall_time); + wall_time = VALUES(wall_time)`) + `; ` var countExecs int diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index ad4c9d13957..86e6144f264 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -987,10 +987,12 @@ CREATE TABLE `host_recovery_key_passwords` ( `updated_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), `pending_encrypted_password` blob, `pending_error_message` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci, + `auto_rotate_at` timestamp(6) NULL DEFAULT NULL, PRIMARY KEY (`host_uuid`), KEY `status` (`status`), KEY `operation_type` (`operation_type`), KEY `deleted` (`deleted`), + KEY `idx_auto_rotate_at` (`auto_rotate_at`), CONSTRAINT `host_recovery_key_passwords_ibfk_1` FOREIGN KEY (`status`) REFERENCES `mdm_delivery_status` (`status`) ON UPDATE CASCADE, CONSTRAINT `host_recovery_key_passwords_ibfk_2` FOREIGN KEY (`operation_type`) REFERENCES `mdm_operation_types` (`operation_type`) ON UPDATE CASCADE ) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; @@ -1805,9 +1807,9 @@ CREATE TABLE `migration_status_tables` ( `is_applied` tinyint(1) NOT NULL, `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=501 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=504 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'),(350,20250124194347,1,'2020-01-01 01:01:01'),(351,20250127162751,1,'2020-01-01 01:01:01'),(352,20250213104005,1,'2020-01-01 01:01:01'),(353,20250214205657,1,'2020-01-01 01:01:01'),(354,20250217093329,1,'2020-01-01 01:01:01'),(355,20250219090511,1,'2020-01-01 01:01:01'),(356,20250219100000,1,'2020-01-01 01:01:01'),(357,20250219142401,1,'2020-01-01 01:01:01'),(358,20250224184002,1,'2020-01-01 01:01:01'),(359,20250225085436,1,'2020-01-01 01:01:01'),(360,20250226000000,1,'2020-01-01 01:01:01'),(361,20250226153445,1,'2020-01-01 01:01:01'),(362,20250304162702,1,'2020-01-01 01:01:01'),(363,20250306144233,1,'2020-01-01 01:01:01'),(364,20250313163430,1,'2020-01-01 01:01:01'),(365,20250317130944,1,'2020-01-01 01:01:01'),(366,20250318165922,1,'2020-01-01 01:01:01'),(367,20250320132525,1,'2020-01-01 01:01:01'),(368,20250320200000,1,'2020-01-01 01:01:01'),(369,20250326161930,1,'2020-01-01 01:01:01'),(370,20250326161931,1,'2020-01-01 01:01:01'),(371,20250331042354,1,'2020-01-01 01:01:01'),(372,20250331154206,1,'2020-01-01 01:01:01'),(373,20250401155831,1,'2020-01-01 01:01:01'),(374,20250408133233,1,'2020-01-01 01:01:01'),(375,20250410104321,1,'2020-01-01 01:01:01'),(376,20250421085116,1,'2020-01-01 01:01:01'),(377,20250422095806,1,'2020-01-01 01:01:01'),(378,20250424153059,1,'2020-01-01 01:01:01'),(379,20250430103833,1,'2020-01-01 01:01:01'),(380,20250430112622,1,'2020-01-01 01:01:01'),(381,20250501162727,1,'2020-01-01 01:01:01'),(382,20250502154517,1,'2020-01-01 01:01:01'),(383,20250502222222,1,'2020-01-01 01:01:01'),(384,20250507170845,1,'2020-01-01 01:01:01'),(385,20250513162912,1,'2020-01-01 01:01:01'),(386,20250519161614,1,'2020-01-01 01:01:01'),(387,20250519170000,1,'2020-01-01 01:01:01'),(388,20250520153848,1,'2020-01-01 01:01:01'),(389,20250528115932,1,'2020-01-01 01:01:01'),(390,20250529102706,1,'2020-01-01 01:01:01'),(391,20250603105558,1,'2020-01-01 01:01:01'),(392,20250609102714,1,'2020-01-01 01:01:01'),(393,20250609112613,1,'2020-01-01 01:01:01'),(394,20250613103810,1,'2020-01-01 01:01:01'),(395,20250616193950,1,'2020-01-01 01:01:01'),(396,20250624140757,1,'2020-01-01 01:01:01'),(397,20250626130239,1,'2020-01-01 01:01:01'),(398,20250629131032,1,'2020-01-01 01:01:01'),(399,20250701155654,1,'2020-01-01 01:01:01'),(400,20250707095725,1,'2020-01-01 01:01:01'),(401,20250716152435,1,'2020-01-01 01:01:01'),(402,20250718091828,1,'2020-01-01 01:01:01'),(403,20250728122229,1,'2020-01-01 01:01:01'),(404,20250731122715,1,'2020-01-01 01:01:01'),(405,20250731151000,1,'2020-01-01 01:01:01'),(406,20250803000000,1,'2020-01-01 01:01:01'),(407,20250805083116,1,'2020-01-01 01:01:01'),(408,20250807140441,1,'2020-01-01 01:01:01'),(409,20250808000000,1,'2020-01-01 01:01:01'),(410,20250811155036,1,'2020-01-01 01:01:01'),(411,20250813205039,1,'2020-01-01 01:01:01'),(412,20250814123333,1,'2020-01-01 01:01:01'),(413,20250815130115,1,'2020-01-01 01:01:01'),(414,20250816115553,1,'2020-01-01 01:01:01'),(415,20250817154557,1,'2020-01-01 01:01:01'),(416,20250825113751,1,'2020-01-01 01:01:01'),(417,20250827113140,1,'2020-01-01 01:01:01'),(418,20250828120836,1,'2020-01-01 01:01:01'),(419,20250902112642,1,'2020-01-01 01:01:01'),(420,20250904091745,1,'2020-01-01 01:01:01'),(421,20250905090000,1,'2020-01-01 01:01:01'),(422,20250922083056,1,'2020-01-01 01:01:01'),(423,20250923120000,1,'2020-01-01 01:01:01'),(424,20250926123048,1,'2020-01-01 01:01:01'),(425,20251015103505,1,'2020-01-01 01:01:01'),(426,20251015103600,1,'2020-01-01 01:01:01'),(427,20251015103700,1,'2020-01-01 01:01:01'),(428,20251015103800,1,'2020-01-01 01:01:01'),(429,20251015103900,1,'2020-01-01 01:01:01'),(430,20251028140000,1,'2020-01-01 01:01:01'),(431,20251028140100,1,'2020-01-01 01:01:01'),(432,20251028140110,1,'2020-01-01 01:01:01'),(433,20251028140200,1,'2020-01-01 01:01:01'),(434,20251028140300,1,'2020-01-01 01:01:01'),(435,20251028140400,1,'2020-01-01 01:01:01'),(436,20251031154558,1,'2020-01-01 01:01:01'),(437,20251103160848,1,'2020-01-01 01:01:01'),(438,20251104112849,1,'2020-01-01 01:01:01'),(439,20251106000000,1,'2020-01-01 01:01:01'),(440,20251107164629,1,'2020-01-01 01:01:01'),(441,20251107170854,1,'2020-01-01 01:01:01'),(442,20251110172137,1,'2020-01-01 01:01:01'),(443,20251111153133,1,'2020-01-01 01:01:01'),(444,20251117020000,1,'2020-01-01 01:01:01'),(445,20251117020100,1,'2020-01-01 01:01:01'),(446,20251117020200,1,'2020-01-01 01:01:01'),(447,20251121100000,1,'2020-01-01 01:01:01'),(448,20251121124239,1,'2020-01-01 01:01:01'),(449,20251124090450,1,'2020-01-01 01:01:01'),(450,20251124135808,1,'2020-01-01 01:01:01'),(451,20251124140138,1,'2020-01-01 01:01:01'),(452,20251124162948,1,'2020-01-01 01:01:01'),(453,20251127113559,1,'2020-01-01 01:01:01'),(454,20251202162232,1,'2020-01-01 01:01:01'),(455,20251203170808,1,'2020-01-01 01:01:01'),(456,20251207050413,1,'2020-01-01 01:01:01'),(457,20251208215800,1,'2020-01-01 01:01:01'),(458,20251209221730,1,'2020-01-01 01:01:01'),(459,20251209221850,1,'2020-01-01 01:01:01'),(460,20251215163721,1,'2020-01-01 01:01:01'),(461,20251217000000,1,'2020-01-01 01:01:01'),(462,20251217120000,1,'2020-01-01 01:01:01'),(463,20251229000000,1,'2020-01-01 01:01:01'),(464,20251229000010,1,'2020-01-01 01:01:01'),(465,20251229000020,1,'2020-01-01 01:01:01'),(466,20260106000000,1,'2020-01-01 01:01:01'),(467,20260108200708,1,'2020-01-01 01:01:01'),(468,20260108214732,1,'2020-01-01 01:01:01'),(469,20260109231821,1,'2020-01-01 01:01:01'),(470,20260113012054,1,'2020-01-01 01:01:01'),(471,20260124200020,1,'2020-01-01 01:01:01'),(472,20260126150840,1,'2020-01-01 01:01:01'),(473,20260126210724,1,'2020-01-01 01:01:01'),(474,20260202151756,1,'2020-01-01 01:01:01'),(475,20260205184907,1,'2020-01-01 01:01:01'),(476,20260210151544,1,'2020-01-01 01:01:01'),(477,20260210155109,1,'2020-01-01 01:01:01'),(478,20260210181120,1,'2020-01-01 01:01:01'),(479,20260211200153,1,'2020-01-01 01:01:01'),(480,20260217141240,1,'2020-01-01 01:01:01'),(481,20260217200906,1,'2020-01-01 01:01:01'),(482,20260218175704,1,'2020-01-01 01:01:01'),(483,20260314120000,1,'2020-01-01 01:01:01'),(484,20260316120000,1,'2020-01-01 01:01:01'),(485,20260316120001,1,'2020-01-01 01:01:01'),(486,20260316120002,1,'2020-01-01 01:01:01'),(487,20260316120003,1,'2020-01-01 01:01:01'),(488,20260316120004,1,'2020-01-01 01:01:01'),(489,20260316120005,1,'2020-01-01 01:01:01'),(490,20260316120006,1,'2020-01-01 01:01:01'),(491,20260316120007,1,'2020-01-01 01:01:01'),(492,20260316120008,1,'2020-01-01 01:01:01'),(493,20260316120009,1,'2020-01-01 01:01:01'),(494,20260316120010,1,'2020-01-01 01:01:01'),(495,20260316120011,1,'2020-01-01 01:01:01'),(496,20260317120000,1,'2020-01-01 01:01:01'),(497,20260318184559,1,'2020-01-01 01:01:01'),(498,20260319120000,1,'2020-01-01 01:01:01'),(499,20260323144117,1,'2020-01-01 01:01:01'),(500,20260324161944,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'),(350,20250124194347,1,'2020-01-01 01:01:01'),(351,20250127162751,1,'2020-01-01 01:01:01'),(352,20250213104005,1,'2020-01-01 01:01:01'),(353,20250214205657,1,'2020-01-01 01:01:01'),(354,20250217093329,1,'2020-01-01 01:01:01'),(355,20250219090511,1,'2020-01-01 01:01:01'),(356,20250219100000,1,'2020-01-01 01:01:01'),(357,20250219142401,1,'2020-01-01 01:01:01'),(358,20250224184002,1,'2020-01-01 01:01:01'),(359,20250225085436,1,'2020-01-01 01:01:01'),(360,20250226000000,1,'2020-01-01 01:01:01'),(361,20250226153445,1,'2020-01-01 01:01:01'),(362,20250304162702,1,'2020-01-01 01:01:01'),(363,20250306144233,1,'2020-01-01 01:01:01'),(364,20250313163430,1,'2020-01-01 01:01:01'),(365,20250317130944,1,'2020-01-01 01:01:01'),(366,20250318165922,1,'2020-01-01 01:01:01'),(367,20250320132525,1,'2020-01-01 01:01:01'),(368,20250320200000,1,'2020-01-01 01:01:01'),(369,20250326161930,1,'2020-01-01 01:01:01'),(370,20250326161931,1,'2020-01-01 01:01:01'),(371,20250331042354,1,'2020-01-01 01:01:01'),(372,20250331154206,1,'2020-01-01 01:01:01'),(373,20250401155831,1,'2020-01-01 01:01:01'),(374,20250408133233,1,'2020-01-01 01:01:01'),(375,20250410104321,1,'2020-01-01 01:01:01'),(376,20250421085116,1,'2020-01-01 01:01:01'),(377,20250422095806,1,'2020-01-01 01:01:01'),(378,20250424153059,1,'2020-01-01 01:01:01'),(379,20250430103833,1,'2020-01-01 01:01:01'),(380,20250430112622,1,'2020-01-01 01:01:01'),(381,20250501162727,1,'2020-01-01 01:01:01'),(382,20250502154517,1,'2020-01-01 01:01:01'),(383,20250502222222,1,'2020-01-01 01:01:01'),(384,20250507170845,1,'2020-01-01 01:01:01'),(385,20250513162912,1,'2020-01-01 01:01:01'),(386,20250519161614,1,'2020-01-01 01:01:01'),(387,20250519170000,1,'2020-01-01 01:01:01'),(388,20250520153848,1,'2020-01-01 01:01:01'),(389,20250528115932,1,'2020-01-01 01:01:01'),(390,20250529102706,1,'2020-01-01 01:01:01'),(391,20250603105558,1,'2020-01-01 01:01:01'),(392,20250609102714,1,'2020-01-01 01:01:01'),(393,20250609112613,1,'2020-01-01 01:01:01'),(394,20250613103810,1,'2020-01-01 01:01:01'),(395,20250616193950,1,'2020-01-01 01:01:01'),(396,20250624140757,1,'2020-01-01 01:01:01'),(397,20250626130239,1,'2020-01-01 01:01:01'),(398,20250629131032,1,'2020-01-01 01:01:01'),(399,20250701155654,1,'2020-01-01 01:01:01'),(400,20250707095725,1,'2020-01-01 01:01:01'),(401,20250716152435,1,'2020-01-01 01:01:01'),(402,20250718091828,1,'2020-01-01 01:01:01'),(403,20250728122229,1,'2020-01-01 01:01:01'),(404,20250731122715,1,'2020-01-01 01:01:01'),(405,20250731151000,1,'2020-01-01 01:01:01'),(406,20250803000000,1,'2020-01-01 01:01:01'),(407,20250805083116,1,'2020-01-01 01:01:01'),(408,20250807140441,1,'2020-01-01 01:01:01'),(409,20250808000000,1,'2020-01-01 01:01:01'),(410,20250811155036,1,'2020-01-01 01:01:01'),(411,20250813205039,1,'2020-01-01 01:01:01'),(412,20250814123333,1,'2020-01-01 01:01:01'),(413,20250815130115,1,'2020-01-01 01:01:01'),(414,20250816115553,1,'2020-01-01 01:01:01'),(415,20250817154557,1,'2020-01-01 01:01:01'),(416,20250825113751,1,'2020-01-01 01:01:01'),(417,20250827113140,1,'2020-01-01 01:01:01'),(418,20250828120836,1,'2020-01-01 01:01:01'),(419,20250902112642,1,'2020-01-01 01:01:01'),(420,20250904091745,1,'2020-01-01 01:01:01'),(421,20250905090000,1,'2020-01-01 01:01:01'),(422,20250922083056,1,'2020-01-01 01:01:01'),(423,20250923120000,1,'2020-01-01 01:01:01'),(424,20250926123048,1,'2020-01-01 01:01:01'),(425,20251015103505,1,'2020-01-01 01:01:01'),(426,20251015103600,1,'2020-01-01 01:01:01'),(427,20251015103700,1,'2020-01-01 01:01:01'),(428,20251015103800,1,'2020-01-01 01:01:01'),(429,20251015103900,1,'2020-01-01 01:01:01'),(430,20251028140000,1,'2020-01-01 01:01:01'),(431,20251028140100,1,'2020-01-01 01:01:01'),(432,20251028140110,1,'2020-01-01 01:01:01'),(433,20251028140200,1,'2020-01-01 01:01:01'),(434,20251028140300,1,'2020-01-01 01:01:01'),(435,20251028140400,1,'2020-01-01 01:01:01'),(436,20251031154558,1,'2020-01-01 01:01:01'),(437,20251103160848,1,'2020-01-01 01:01:01'),(438,20251104112849,1,'2020-01-01 01:01:01'),(439,20251106000000,1,'2020-01-01 01:01:01'),(440,20251107164629,1,'2020-01-01 01:01:01'),(441,20251107170854,1,'2020-01-01 01:01:01'),(442,20251110172137,1,'2020-01-01 01:01:01'),(443,20251111153133,1,'2020-01-01 01:01:01'),(444,20251117020000,1,'2020-01-01 01:01:01'),(445,20251117020100,1,'2020-01-01 01:01:01'),(446,20251117020200,1,'2020-01-01 01:01:01'),(447,20251121100000,1,'2020-01-01 01:01:01'),(448,20251121124239,1,'2020-01-01 01:01:01'),(449,20251124090450,1,'2020-01-01 01:01:01'),(450,20251124135808,1,'2020-01-01 01:01:01'),(451,20251124140138,1,'2020-01-01 01:01:01'),(452,20251124162948,1,'2020-01-01 01:01:01'),(453,20251127113559,1,'2020-01-01 01:01:01'),(454,20251202162232,1,'2020-01-01 01:01:01'),(455,20251203170808,1,'2020-01-01 01:01:01'),(456,20251207050413,1,'2020-01-01 01:01:01'),(457,20251208215800,1,'2020-01-01 01:01:01'),(458,20251209221730,1,'2020-01-01 01:01:01'),(459,20251209221850,1,'2020-01-01 01:01:01'),(460,20251215163721,1,'2020-01-01 01:01:01'),(461,20251217000000,1,'2020-01-01 01:01:01'),(462,20251217120000,1,'2020-01-01 01:01:01'),(463,20251229000000,1,'2020-01-01 01:01:01'),(464,20251229000010,1,'2020-01-01 01:01:01'),(465,20251229000020,1,'2020-01-01 01:01:01'),(466,20260106000000,1,'2020-01-01 01:01:01'),(467,20260108200708,1,'2020-01-01 01:01:01'),(468,20260108214732,1,'2020-01-01 01:01:01'),(469,20260109231821,1,'2020-01-01 01:01:01'),(470,20260113012054,1,'2020-01-01 01:01:01'),(471,20260124200020,1,'2020-01-01 01:01:01'),(472,20260126150840,1,'2020-01-01 01:01:01'),(473,20260126210724,1,'2020-01-01 01:01:01'),(474,20260202151756,1,'2020-01-01 01:01:01'),(475,20260205184907,1,'2020-01-01 01:01:01'),(476,20260210151544,1,'2020-01-01 01:01:01'),(477,20260210155109,1,'2020-01-01 01:01:01'),(478,20260210181120,1,'2020-01-01 01:01:01'),(479,20260211200153,1,'2020-01-01 01:01:01'),(480,20260217141240,1,'2020-01-01 01:01:01'),(481,20260217200906,1,'2020-01-01 01:01:01'),(482,20260218175704,1,'2020-01-01 01:01:01'),(483,20260314120000,1,'2020-01-01 01:01:01'),(484,20260316120000,1,'2020-01-01 01:01:01'),(485,20260316120001,1,'2020-01-01 01:01:01'),(486,20260316120002,1,'2020-01-01 01:01:01'),(487,20260316120003,1,'2020-01-01 01:01:01'),(488,20260316120004,1,'2020-01-01 01:01:01'),(489,20260316120005,1,'2020-01-01 01:01:01'),(490,20260316120006,1,'2020-01-01 01:01:01'),(491,20260316120007,1,'2020-01-01 01:01:01'),(492,20260316120008,1,'2020-01-01 01:01:01'),(493,20260316120009,1,'2020-01-01 01:01:01'),(494,20260316120010,1,'2020-01-01 01:01:01'),(495,20260316120011,1,'2020-01-01 01:01:01'),(496,20260317120000,1,'2020-01-01 01:01:01'),(497,20260318184559,1,'2020-01-01 01:01:01'),(498,20260319120000,1,'2020-01-01 01:01:01'),(499,20260323144117,1,'2020-01-01 01:01:01'),(500,20260324161944,1,'2020-01-01 01:01:01'),(501,20260324223334,1,'2020-01-01 01:01:01'),(502,20260326131501,1,'2020-01-01 01:01:01'),(503,20260326210603,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `mobile_device_management_solutions` ( @@ -2315,8 +2317,10 @@ CREATE TABLE `query_results` ( `error` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci, `last_fetched` timestamp NOT NULL, `data` json DEFAULT NULL, + `has_data` tinyint(1) GENERATED ALWAYS AS ((`data` is not null)) VIRTUAL, PRIMARY KEY (`id`), - KEY `idx_query_id_host_id_last_fetched` (`query_id`,`host_id`,`last_fetched`) + KEY `idx_query_id_host_id_last_fetched` (`query_id`,`host_id`,`last_fetched`), + KEY `idx_query_id_has_data_host_id_last_fetched` (`query_id`,`has_data`,`host_id`,`last_fetched`) ) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; diff --git a/server/datastore/mysql/scim.go b/server/datastore/mysql/scim.go index cf3fa525095..b5221b30e06 100644 --- a/server/datastore/mysql/scim.go +++ b/server/datastore/mysql/scim.go @@ -32,8 +32,7 @@ func (ds *Datastore) CreateScimUser(ctx context.Context, user *fleet.ScimUser) ( INSERT INTO scim_users ( external_id, user_name, given_name, family_name, department, active ) VALUES (?, ?, ?, ?, ?, ?)` - result, err := tx.ExecContext( - ctx, + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertUserQuery, user.ExternalID, user.UserName, @@ -43,16 +42,12 @@ func (ds *Datastore) CreateScimUser(ctx context.Context, user *fleet.ScimUser) ( user.Active, ) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return ctxerr.Wrap(ctx, alreadyExists("ScimUser", user.UserName), "insert scim user") } return ctxerr.Wrap(ctx, err, "insert scim user") } - id, err := result.LastInsertId() - if err != nil { - return ctxerr.Wrap(ctx, err, "insert scim user last insert id") - } user.ID = uint(id) // nolint:gosec // dismiss G115 userID = user.ID @@ -309,7 +304,7 @@ func (ds *Datastore) ReplaceScimUser(ctx context.Context, user *fleet.ScimUser) user.ID, ) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return ctxerr.Wrap(ctx, alreadyExists("ScimUser", user.UserName), "update scim user") } return ctxerr.Wrap(ctx, err, "update scim user") @@ -651,8 +646,7 @@ func (ds *Datastore) CreateScimGroup(ctx context.Context, group *fleet.ScimGroup INSERT INTO scim_groups ( external_id, display_name ) VALUES (?, ?)` - result, err := tx.ExecContext( - ctx, + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertGroupQuery, group.ExternalID, group.DisplayName, @@ -661,10 +655,6 @@ func (ds *Datastore) CreateScimGroup(ctx context.Context, group *fleet.ScimGroup return ctxerr.Wrap(ctx, err, "insert scim group") } - id, err := result.LastInsertId() - if err != nil { - return ctxerr.Wrap(ctx, err, "insert scim group last insert id") - } group.ID = uint(id) // nolint:gosec // dismiss G115 groupID = group.ID diff --git a/server/datastore/mysql/scripts.go b/server/datastore/mysql/scripts.go index cd8bbca8b29..012f0945441 100644 --- a/server/datastore/mysql/scripts.go +++ b/server/datastore/mysql/scripts.go @@ -26,12 +26,11 @@ func (ds *Datastore) NewHostScriptExecutionRequest(ctx context.Context, request var err error if request.ScriptContentID == 0 { // then we are doing a sync execution, so create the contents first - scRes, err := insertScriptContents(ctx, tx, request.ScriptContents) + id, err := insertScriptContents(ctx, tx, ds.dialect, request.ScriptContents) if err != nil { return err } - id, _ := scRes.LastInsertId() request.ScriptContentID = uint(id) //nolint:gosec // dismiss G115 } res, err = ds.newHostScriptExecutionRequest(ctx, tx, request, false) @@ -79,8 +78,8 @@ INSERT INTO upcoming_activities VALUES (?, ?, ?, ?, 'script', ?, JSON_OBJECT( - 'sync_request', ?, - 'is_internal', ?, + 'sync_request', CAST(? AS UNSIGNED), + 'is_internal', CAST(? AS UNSIGNED), 'user', (SELECT JSON_OBJECT('name', name, 'email', email, 'gravatar_url', gravatar_url) FROM users WHERE id = ?) ) )` @@ -94,21 +93,29 @@ VALUES ) execID := uuid.New().String() - result, err := tx.ExecContext(ctx, insUAStmt, + // Convert booleans to int for JSON_OBJECT compatibility with PG's jsonb_build_object, + // which needs typed parameters. CAST(? AS UNSIGNED) → CAST($N AS integer) on PG. + syncRequestInt := 0 + if request.SyncRequest { + syncRequestInt = 1 + } + isInternalInt := 0 + if isInternal { + isInternalInt = 1 + } + activityID, err := insertAndGetIDTx(ctx, tx, ds.dialect, insUAStmt, request.HostID, request.Priority(), request.UserID, request.PolicyID != nil, // fleet-initiated if request is via a policy failure execID, - request.SyncRequest, - isInternal, + syncRequestInt, + isInternalInt, request.UserID, ) if err != nil { return "", 0, ctxerr.Wrap(ctx, err, "new script upcoming activity") } - - activityID, _ := result.LastInsertId() _, err = tx.ExecContext(ctx, insSUAStmt, activityID, request.ScriptID, @@ -292,7 +299,7 @@ func (ds *Datastore) listUpcomingHostScriptExecutions(ctx context.Context, hostI extraWhere := "" if onlyShowInternal { // software_uninstalls are implicitly internal - extraWhere = " AND COALESCE(ua.payload->'$.is_internal', 1) = 1" + extraWhere = " AND COALESCE(ua.payload->>'$.is_internal', '1') = '1'" } if onlyReadyToExecute { extraWhere += " AND ua.activated_at IS NOT NULL" @@ -408,11 +415,10 @@ func (ds *Datastore) getHostScriptExecutionResultDB(ctx context.Context, q sqlx. LEFT JOIN batch_activity_host_results bahr ON hsr.execution_id = bahr.host_execution_id JOIN - script_contents sc + script_contents sc ON hsr.script_content_id = sc.id %s WHERE - hsr.execution_id = ? AND - hsr.script_content_id = sc.id + hsr.execution_id = ? %s `, uninstallCondition, canceledCondition) @@ -431,7 +437,7 @@ func (ds *Datastore) getHostScriptExecutionResultDB(ctx context.Context, q sqlx. NULL as timeout, ua.created_at, ua.user_id, - COALESCE(ua.payload->'$.sync_request', 0) as sync_request, + COALESCE(ua.payload->>'$.sync_request', '0') = '1' as sync_request, NULL as host_deleted_at, sua.setup_experience_script_id, 0 as canceled, @@ -487,26 +493,22 @@ func (ds *Datastore) CountHostScriptAttempts(ctx context.Context, hostID, script } func (ds *Datastore) NewScript(ctx context.Context, script *fleet.Script) (*fleet.Script, error) { - var res sql.Result + var scriptID int64 err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - var err error - // first insert script contents - scRes, err := insertScriptContents(ctx, tx, script.ScriptContents) + contentID, err := insertScriptContents(ctx, tx, ds.dialect, script.ScriptContents) if err != nil { return err } - id, _ := scRes.LastInsertId() // then create the script entity - res, err = insertScript(ctx, tx, script, uint(id)) //nolint:gosec // dismiss G115 + scriptID, err = insertScript(ctx, tx, ds.dialect, script, uint(contentID)) //nolint:gosec // dismiss G115 return err }) if err != nil { return nil, err } - id, _ := res.LastInsertId() - return ds.getScriptDB(ctx, ds.writer(ctx), uint(id)) //nolint:gosec // dismiss G115 + return ds.getScriptDB(ctx, ds.writer(ctx), uint(scriptID)) //nolint:gosec // dismiss G115 } func (ds *Datastore) UpdateScriptContents(ctx context.Context, scriptID uint, scriptContents string) (*fleet.Script, error) { @@ -520,11 +522,10 @@ func (ds *Datastore) UpdateScriptContents(ctx context.Context, scriptID uint, sc } // Insert or get existing content (insertScriptContents handles deduplication) - scRes, err := insertScriptContents(ctx, tx, scriptContents) + newContentID, err := insertScriptContents(ctx, tx, ds.dialect, scriptContents) if err != nil { return ctxerr.Wrap(ctx, err, "inserting/getting script contents") } - newContentID, _ := scRes.LastInsertId() // Update the script to point to the new content if newContentID != oldContentID { @@ -622,7 +623,7 @@ func (ds *Datastore) resetScriptPolicyAutomationAttempts(ctx context.Context, db return nil } -func insertScript(ctx context.Context, tx sqlx.ExtContext, script *fleet.Script, scriptContentsID uint) (sql.Result, error) { +func insertScript(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, script *fleet.Script, scriptContentsID uint) (int64, error) { const insertStmt = ` INSERT INTO scripts ( @@ -635,7 +636,7 @@ VALUES if script.TeamID != nil { globalOrTeamID = *script.TeamID } - res, err := tx.ExecContext(ctx, insertStmt, + id, err := insertAndGetIDTx(ctx, tx, dialect, insertStmt, script.TeamID, globalOrTeamID, script.Name, scriptContentsID) if err != nil { if IsDuplicate(err) { @@ -645,29 +646,27 @@ VALUES // team does not exist err = foreignKey("scripts", fmt.Sprintf("team_id=%v", script.TeamID)) } - return nil, ctxerr.Wrap(ctx, err, "insert script") + return 0, ctxerr.Wrap(ctx, err, "insert script") } - return res, nil + return id, nil } -func insertScriptContents(ctx context.Context, tx sqlx.ExtContext, contents string) (sql.Result, error) { - const insertStmt = ` +func insertScriptContents(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, contents string) (int64, error) { + insertStmt := ` INSERT INTO script_contents ( md5_checksum, contents ) VALUES (UNHEX(?),?) -ON DUPLICATE KEY UPDATE - id=LAST_INSERT_ID(id) - ` +` + dialect.OnDuplicateKey("md5_checksum", "id=LAST_INSERT_ID(id)") md5Checksum := md5ChecksumScriptContent(contents) - res, err := tx.ExecContext(ctx, insertStmt, md5Checksum, contents) + id, err := insertAndGetIDTx(ctx, tx, dialect, insertStmt, md5Checksum, contents) if err != nil { - return nil, ctxerr.Wrap(ctx, err, "insert script contents") + return 0, ctxerr.Wrap(ctx, err, "insert script contents") } - return res, nil + return id, nil } func md5ChecksumScriptContent(s string) string { @@ -810,7 +809,7 @@ func (ds *Datastore) DeleteScript(ctx context.Context, id uint) error { WHERE sua.script_id = ? AND ua.activity_type = 'script' AND ua.activated_at IS NOT NULL AND - (ua.payload->'$.sync_request' = 0 OR + (COALESCE(ua.payload->>'$.sync_request', '0') = '0' OR ua.created_at >= NOW() - INTERVAL ? SECOND)` var affectedHosts []uint if err := sqlx.SelectContext(ctx, tx, &affectedHosts, loadAffectedHostsStmt, @@ -825,7 +824,7 @@ func (ds *Datastore) DeleteScript(ctx context.Context, id uint) error { ON upcoming_activities.id = sua.upcoming_activity_id WHERE sua.script_id = ? AND upcoming_activities.activity_type = 'script' AND - (upcoming_activities.payload->'$.sync_request' = 0 OR + (COALESCE(upcoming_activities.payload->>'$.sync_request', '0') = '0' OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) `, id, int(constants.MaxServerWaitTime.Seconds()), @@ -836,7 +835,7 @@ func (ds *Datastore) DeleteScript(ctx context.Context, id uint) error { _, err = tx.ExecContext(ctx, `DELETE FROM scripts WHERE id = ?`, id) if err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { // Check if the script is referenced by a policy automation. var count int if err := sqlx.GetContext(ctx, tx, &count, `SELECT COUNT(*) FROM policies WHERE script_id = ?`, id); err != nil { @@ -1171,7 +1170,7 @@ WHERE WHERE ua.activity_type = 'script' AND ua.activated_at IS NOT NULL - AND (ua.payload->'$.sync_request' = 0 OR ua.created_at >= NOW() - INTERVAL ? SECOND) + AND (COALESCE(ua.payload->>'$.sync_request', '0') = '0' OR ua.created_at >= NOW() - INTERVAL ? SECOND) AND sua.script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ?)` const clearAllPendingExecutionsUA = `DELETE FROM upcoming_activities @@ -1181,7 +1180,7 @@ WHERE ON upcoming_activities.id = sua.upcoming_activity_id WHERE upcoming_activities.activity_type = 'script' - AND (upcoming_activities.payload->'$.sync_request' = 0 OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) + AND (COALESCE(upcoming_activities.payload->>'$.sync_request', '0') = '0' OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) AND sua.script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ?)` const unsetScriptsNotInListFromPolicies = ` @@ -1211,7 +1210,7 @@ WHERE WHERE ua.activity_type = 'script' AND ua.activated_at IS NOT NULL - AND (ua.payload->'$.sync_request' = 0 OR ua.created_at >= NOW() - INTERVAL ? SECOND) + AND (COALESCE(ua.payload->>'$.sync_request', '0') = '0' OR ua.created_at >= NOW() - INTERVAL ? SECOND) AND sua.script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ? AND name NOT IN (?))` const clearPendingExecutionsNotInListUA = `DELETE FROM upcoming_activities @@ -1221,19 +1220,17 @@ WHERE ON upcoming_activities.id = sua.upcoming_activity_id WHERE upcoming_activities.activity_type = 'script' - AND (upcoming_activities.payload->'$.sync_request' = 0 OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) + AND (COALESCE(upcoming_activities.payload->>'$.sync_request', '0') = '0' OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) AND sua.script_id IN (SELECT id FROM scripts WHERE global_or_team_id = ? AND name NOT IN (?))` - const insertNewOrEditedScript = ` + insertNewOrEditedScript := ` INSERT INTO scripts ( team_id, global_or_team_id, name, script_content_id ) VALUES (?, ?, ?, ?) -ON DUPLICATE KEY UPDATE - script_content_id = VALUES(script_content_id), id=LAST_INSERT_ID(id) -` +` + ds.dialect.OnDuplicateKey("id", "script_content_id = VALUES(script_content_id), id=LAST_INSERT_ID(id)") const clearPendingExecutionsWithObsoleteScriptHSR = `DELETE FROM host_script_results WHERE exit_code IS NULL AND (sync_request = 0 OR created_at >= NOW() - INTERVAL ? SECOND) @@ -1249,7 +1246,7 @@ ON DUPLICATE KEY UPDATE WHERE ua.activity_type = 'script' AND ua.activated_at IS NOT NULL - AND (ua.payload->'$.sync_request' = 0 OR ua.created_at >= NOW() - INTERVAL ? SECOND) + AND (COALESCE(ua.payload->>'$.sync_request', '0') = '0' OR ua.created_at >= NOW() - INTERVAL ? SECOND) AND sua.script_id = ? AND sua.script_content_id != ?` const clearPendingExecutionsWithObsoleteScriptUA = `DELETE FROM upcoming_activities @@ -1259,7 +1256,7 @@ ON DUPLICATE KEY UPDATE ON upcoming_activities.id = sua.upcoming_activity_id WHERE upcoming_activities.activity_type = 'script' - AND (upcoming_activities.payload->'$.sync_request' = 0 OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) + AND (COALESCE(upcoming_activities.payload->>'$.sync_request', '0') = '0' OR upcoming_activities.created_at >= NOW() - INTERVAL ? SECOND) AND sua.script_id = ? AND sua.script_content_id != ?` const loadInsertedScripts = `SELECT id, team_id, name FROM scripts WHERE global_or_team_id = ?` @@ -1382,16 +1379,14 @@ ON DUPLICATE KEY UPDATE // insert the new scripts and the ones that have changed for _, s := range incomingScripts { - scRes, err := insertScriptContents(ctx, tx, s.ScriptContents) + contentID, err := insertScriptContents(ctx, tx, ds.dialect, s.ScriptContents) if err != nil { return ctxerr.Wrapf(ctx, err, "inserting script contents for script with name %q", s.Name) } - contentID, _ := scRes.LastInsertId() - insertRes, err := tx.ExecContext(ctx, insertNewOrEditedScript, tmID, globalOrTeamID, s.Name, uint(contentID)) //nolint:gosec // dismiss G115 + scriptID, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertNewOrEditedScript, tmID, globalOrTeamID, s.Name, uint(contentID)) //nolint:gosec // dismiss G115 if err != nil { return ctxerr.Wrapf(ctx, err, "insert new/edited script with name %q", s.Name) } - scriptID, _ := insertRes.LastInsertId() if _, err := tx.ExecContext(ctx, clearPendingExecutionsWithObsoleteScriptHSR, int(constants.MaxServerWaitTime.Seconds()), scriptID, contentID); err != nil { return ctxerr.Wrapf(ctx, err, "clear obsolete pending script executions with name %q", s.Name) @@ -2085,12 +2080,11 @@ func (ds *Datastore) LockHostViaScript(ctx context.Context, request *fleet.HostS return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { var err error - scRes, err := insertScriptContents(ctx, tx, request.ScriptContents) + id, err := insertScriptContents(ctx, tx, ds.dialect, request.ScriptContents) if err != nil { return err } - id, _ := scRes.LastInsertId() request.ScriptContentID = uint(id) //nolint:gosec // dismiss G115 res, err = ds.newHostScriptExecutionRequest(ctx, tx, request, true) @@ -2103,7 +2097,7 @@ func (ds *Datastore) LockHostViaScript(ctx context.Context, request *fleet.HostS // it is pending execution. The host's state should be updated to "locked" // only when the script execution is successfully completed, and then any // unlock or wipe references should be cleared. - const stmt = ` + stmt := ` INSERT INTO host_mdm_actions ( host_id, @@ -2111,9 +2105,7 @@ func (ds *Datastore) LockHostViaScript(ctx context.Context, request *fleet.HostS fleet_platform ) VALUES (?,?,?) - ON DUPLICATE KEY UPDATE - lock_ref = VALUES(lock_ref) - ` + ` + ds.dialect.OnDuplicateKey("host_id", `lock_ref = VALUES(lock_ref)`) _, err = tx.ExecContext(ctx, stmt, request.HostID, @@ -2135,12 +2127,11 @@ func (ds *Datastore) UnlockHostViaScript(ctx context.Context, request *fleet.Hos return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { var err error - scRes, err := insertScriptContents(ctx, tx, request.ScriptContents) + id, err := insertScriptContents(ctx, tx, ds.dialect, request.ScriptContents) if err != nil { return err } - id, _ := scRes.LastInsertId() request.ScriptContentID = uint(id) //nolint:gosec // dismiss G115 res, err = ds.newHostScriptExecutionRequest(ctx, tx, request, true) @@ -2153,7 +2144,7 @@ func (ds *Datastore) UnlockHostViaScript(ctx context.Context, request *fleet.Hos // recorded, it is pending execution. The host's state should be updated to // "unlocked" only when the script execution is successfully completed, and // then any lock or wipe references should be cleared. - const stmt = ` + stmt := ` INSERT INTO host_mdm_actions ( host_id, @@ -2161,10 +2152,8 @@ func (ds *Datastore) UnlockHostViaScript(ctx context.Context, request *fleet.Hos fleet_platform ) VALUES (?,?,?) - ON DUPLICATE KEY UPDATE - unlock_ref = VALUES(unlock_ref), - unlock_pin = NULL - ` + ` + ds.dialect.OnDuplicateKey("host_id", `unlock_ref = VALUES(unlock_ref), + unlock_pin = NULL`) _, err = tx.ExecContext(ctx, stmt, request.HostID, @@ -2186,12 +2175,11 @@ func (ds *Datastore) WipeHostViaScript(ctx context.Context, request *fleet.HostS return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { var err error - scRes, err := insertScriptContents(ctx, tx, request.ScriptContents) + id, err := insertScriptContents(ctx, tx, ds.dialect, request.ScriptContents) if err != nil { return err } - id, _ := scRes.LastInsertId() request.ScriptContentID = uint(id) //nolint:gosec // dismiss G115 res, err = ds.newHostScriptExecutionRequest(ctx, tx, request, true) @@ -2203,7 +2191,7 @@ func (ds *Datastore) WipeHostViaScript(ctx context.Context, request *fleet.HostS // point in time, this is just a request to wipe the host that is recorded, // it is pending execution, so if it was locked, it is still locked (so the // lock_ref info must still be there). - const stmt = ` + stmt := ` INSERT INTO host_mdm_actions ( host_id, @@ -2211,9 +2199,7 @@ func (ds *Datastore) WipeHostViaScript(ctx context.Context, request *fleet.HostS fleet_platform ) VALUES (?,?,?) - ON DUPLICATE KEY UPDATE - wipe_ref = VALUES(wipe_ref) - ` + ` + ds.dialect.OnDuplicateKey("host_id", `wipe_ref = VALUES(wipe_ref)`) _, err = tx.ExecContext(ctx, stmt, request.HostID, @@ -2229,7 +2215,7 @@ func (ds *Datastore) WipeHostViaScript(ctx context.Context, request *fleet.HostS } func (ds *Datastore) UnlockHostManually(ctx context.Context, hostID uint, hostFleetPlatform string, ts time.Time) error { - const stmt = ` + stmt := ` INSERT INTO host_mdm_actions ( host_id, @@ -2237,10 +2223,8 @@ func (ds *Datastore) UnlockHostManually(ctx context.Context, hostID uint, hostFl fleet_platform ) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE - -- do not overwrite if a value is already set - unlock_ref = IF(unlock_ref IS NULL, VALUES(unlock_ref), unlock_ref) - ` + ` + ds.dialect.OnDuplicateKey("host_id", `-- do not overwrite if a value is already set + unlock_ref = IF(unlock_ref IS NULL, VALUES(unlock_ref), unlock_ref)`) // for macOS, the unlock_ref is just the timestamp at which the user first // requested to unlock the host. This then indicates in the host's status // that it's pending an unlock (which requires manual intervention by @@ -2474,8 +2458,8 @@ func (ds *Datastore) batchExecuteScript(ctx context.Context, userID *uint, scrip _, err := tx.ExecContext( ctx, - `INSERT INTO batch_activities (execution_id, script_id, status, activity_type, num_targeted, started_at) VALUES (?, ?, ?, ?, ?, NOW()) - ON DUPLICATE KEY UPDATE status = VALUES(status), started_at = VALUES(started_at)`, + `INSERT INTO batch_activities (execution_id, script_id, status, activity_type, num_targeted, started_at) VALUES (?, ?, ?, ?, ?, NOW()) `+ + ds.dialect.OnDuplicateKey("id", "status = VALUES(status), started_at = VALUES(started_at)"), batchExecID, script.ID, fleet.ScheduledBatchExecutionStarted, @@ -2507,7 +2491,7 @@ func (ds *Datastore) batchExecuteScript(ctx context.Context, userID *uint, scrip :host_id, :host_execution_id, :error - ) ON DUPLICATE KEY UPDATE host_execution_id = VALUES(host_execution_id), error = VALUES(error)` + ) ` + ds.dialect.OnDuplicateKey("id", "host_execution_id = VALUES(host_execution_id), error = VALUES(error)") if _, err := sqlx.NamedExecContext(ctx, tx, insertStmt, args); err != nil { return ctxerr.Wrap(ctx, err, "associating script executions with batch job") diff --git a/server/datastore/mysql/secret_variables.go b/server/datastore/mysql/secret_variables.go index 0dcf041d4af..ea8aa77afa4 100644 --- a/server/datastore/mysql/secret_variables.go +++ b/server/datastore/mysql/secret_variables.go @@ -103,17 +103,16 @@ func (ds *Datastore) CreateSecretVariable(ctx context.Context, name string, valu if err != nil { return 0, ctxerr.Wrap(ctx, err, "encrypt secret value for insert with server private key") } - res, err := ds.writer(ctx).ExecContext(ctx, + id_, err := ds.insertAndGetID(ctx, ds.writer(ctx), `INSERT INTO secret_variables (name, value) VALUES (?, ?)`, name, valueEncrypted, ) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { return 0, ctxerr.Wrap(ctx, alreadyExists("name", name), "found duplicate") } return 0, ctxerr.Wrap(ctx, err, "insert secret variable") } - id_, _ := res.LastInsertId() return uint(id_), nil //nolint:gosec // dismiss G115 } diff --git a/server/datastore/mysql/sessions.go b/server/datastore/mysql/sessions.go index d10f7b8f0fa..328806c2e53 100644 --- a/server/datastore/mysql/sessions.go +++ b/server/datastore/mysql/sessions.go @@ -152,12 +152,11 @@ func (ds *Datastore) makeSessionInTransaction(ctx context.Context, tx sqlx.ExtCo ) VALUES(?,?) ` - result, err := tx.ExecContext(ctx, sqlStatement, userID, sessionKey) + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, sqlStatement, userID, sessionKey) if err != nil { return nil, ctxerr.Wrap(ctx, err, "saving session") } - id, _ := result.LastInsertId() // cannot fail with the mysql driver return ds.sessionByID(ctx, tx, uint(id)) //nolint:gosec // dismiss G115 } diff --git a/server/datastore/mysql/setup_experience.go b/server/datastore/mysql/setup_experience.go index 8804699f097..ee1d446e15a 100644 --- a/server/datastore/mysql/setup_experience.go +++ b/server/datastore/mysql/setup_experience.go @@ -249,7 +249,7 @@ WHERE global_or_team_id = ?` // Set setup experience on Apple hosts only if they have something configured. if fleetPlatform == "darwin" || fleetPlatform == "ios" || fleetPlatform == "ipados" { if totalInsertions > 0 { - if err := setHostAwaitingConfiguration(ctx, tx, hostUUID, true); err != nil { + if err := setHostAwaitingConfiguration(ctx, tx, ds.dialect, hostUUID, true); err != nil { return ctxerr.Wrap(ctx, err, "setting host awaiting configuration to true") } } @@ -733,14 +733,11 @@ WHERE func (ds *Datastore) SetSetupExperienceScript(ctx context.Context, script *fleet.Script) error { err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - var err error - // first insert script contents - scRes, err := insertScriptContents(ctx, tx, script.ScriptContents) + id, err := insertScriptContents(ctx, tx, ds.dialect, script.ScriptContents) if err != nil { return err } - id, _ := scRes.LastInsertId() // This clause allows for PUT semantics. The basic idea is: // - no existing setup script -> go through the usual insert logic @@ -824,17 +821,15 @@ func (ds *Datastore) deleteSetupExperienceScript(ctx context.Context, tx sqlx.Ex func (ds *Datastore) SetHostAwaitingConfiguration(ctx context.Context, hostUUID string, awaitingConfiguration bool) error { return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return setHostAwaitingConfiguration(ctx, tx, hostUUID, awaitingConfiguration) + return setHostAwaitingConfiguration(ctx, tx, ds.dialect, hostUUID, awaitingConfiguration) }) } -func setHostAwaitingConfiguration(ctx context.Context, tx sqlx.ExtContext, hostUUID string, awaitingConfiguration bool) error { - const stmt = ` +func setHostAwaitingConfiguration(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, hostUUID string, awaitingConfiguration bool) error { + stmt := ` INSERT INTO host_mdm_apple_awaiting_configuration (host_uuid, awaiting_configuration) VALUES (?, ?) -ON DUPLICATE KEY UPDATE - awaiting_configuration = VALUES(awaiting_configuration) - ` +` + dialect.OnDuplicateKey("host_uuid", "awaiting_configuration = VALUES(awaiting_configuration)") _, err := tx.ExecContext(ctx, stmt, hostUUID, awaitingConfiguration) if err != nil { diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 5f49ab6ebdb..dcfc7d484bf 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -29,7 +29,7 @@ import ( type softwareSummary struct { ID uint `db:"id"` - Checksum string `db:"checksum"` + Checksum []byte `db:"checksum"` Name string `db:"name"` TitleID *uint `db:"title_id"` BundleIdentifier *string `db:"bundle_identifier"` @@ -489,7 +489,7 @@ func (ds *Datastore) applyChangesForNewSoftwareDB( return err } - if err = updateSoftwareUpdatedAt(ctx, tx, hostID); err != nil { + if err = updateSoftwareUpdatedAt(ctx, tx, ds.dialect, hostID); err != nil { return err } return nil @@ -605,9 +605,9 @@ func (ds *Datastore) getExistingSoftware( } if len(newChecksumsToSoftware) > 0 { - sliceOfNewSWChecksums := make([]string, 0, len(newChecksumsToSoftware)) + sliceOfNewSWChecksums := make([][]byte, 0, len(newChecksumsToSoftware)) for checksum := range newChecksumsToSoftware { - sliceOfNewSWChecksums = append(sliceOfNewSWChecksums, checksum) + sliceOfNewSWChecksums = append(sliceOfNewSWChecksums, []byte(checksum)) } // We use the replica DB for retrieval to minimize the traffic to the writer DB. // It is OK if the software is not found in the replica DB, because we will then attempt to insert it in the writer DB. @@ -617,14 +617,14 @@ func (ds *Datastore) getExistingSoftware( } for _, currentSoftwareSummary := range currentSoftwareSummaries { - _, ok := newChecksumsToSoftware[currentSoftwareSummary.Checksum] + _, ok := newChecksumsToSoftware[string(currentSoftwareSummary.Checksum)] if !ok { // This should never happen. If it does, we have a bug. return nil, nil, nil, ctxerr.New( - ctx, fmt.Sprintf("current software: software not found for checksum %s", hex.EncodeToString([]byte(currentSoftwareSummary.Checksum))), + ctx, fmt.Sprintf("current software: software not found for checksum %s", hex.EncodeToString(currentSoftwareSummary.Checksum)), ) } - delete(setOfNewSWChecksums, currentSoftwareSummary.Checksum) + delete(setOfNewSWChecksums, string(currentSoftwareSummary.Checksum)) } } @@ -878,7 +878,7 @@ func (ds *Datastore) preInsertSoftwareInventory( existingSet := make(map[string]struct{}, len(existingSoftwareSummaries)) for _, es := range existingSoftwareSummaries { - existingSet[es.Checksum] = struct{}{} + existingSet[string(es.Checksum)] = struct{}{} } for checksum, sw := range incomingSoftwareByChecksum { @@ -925,22 +925,6 @@ func (ds *Datastore) preInsertSoftwareInventory( } } - // Fetch FMA canonical names to override osquery-reported names for macOS apps. - // This ensures software titles use consistent names (e.g., "Microsoft Visual Studio Code" - // instead of "Code" which is what osquery reports for VS Code). - // Note: This call is made from the base datastore so it bypasses the cached_mysql layer. - // The query is simple (SELECT from the small fleet_maintained_apps table) so this is acceptable. - // The cached_mysql layer still caches this method for other callers (e.g., API endpoints). - fmaNames, fmaErr := ds.GetFMANamesByIdentifier(ctx) - if fmaErr != nil { - // Log but don't fail - we can still use osquery-reported names. - // A nil map is safe here since Go's map access on nil returns the zero value. - if ds.logger != nil { - ds.logger.WarnContext(ctx, "failed to get FMA names by identifier", "err", fmaErr) - } - fmaNames = nil - } - // Process in smaller batches to reduce lock time err := common_mysql.BatchProcessSimple(keys, softwareInventoryInsertBatchSize, func(batchKeys []string) error { batchSoftware := make(map[string]fleet.Software, len(batchKeys)) @@ -957,19 +941,13 @@ func (ds *Datastore) preInsertSoftwareInventory( // there is not an existing software title corresponding to this incoming software version newTitleName := sw.Name if sw.BundleIdentifier != "" { - // First check if there's an FMA with this bundle identifier - use its canonical name - if fmaName, ok := fmaNames[sw.BundleIdentifier]; ok { - newTitleName = fmaName - } else { - // Fall back to computed best name from osquery reports - key := titleKey{ - bundleID: sw.BundleIdentifier, - source: sw.Source, - extensionFor: sw.ExtensionFor, - } - if computedName, exists := bestTitleNames[key]; exists { - newTitleName = computedName - } + key := titleKey{ + bundleID: sw.BundleIdentifier, + source: sw.Source, + extensionFor: sw.ExtensionFor, + } + if computedName, exists := bestTitleNames[key]; exists { + newTitleName = computedName } } @@ -1024,7 +1002,7 @@ func (ds *Datastore) preInsertSoftwareInventory( // Insert software titles const numberOfArgsPerSoftwareTitles = 7 titlesValues := strings.TrimSuffix(strings.Repeat("(?,?,?,?,?,?,?),", len(uniqueTitlesToInsert)), ",") - titlesStmt := fmt.Sprintf("INSERT IGNORE INTO software_titles (name, source, extension_for, bundle_identifier, is_kernel, application_id, upgrade_code) VALUES %s", titlesValues) + titlesStmt := fmt.Sprintf(ds.dialect.InsertIgnoreInto()+" software_titles (name, source, extension_for, bundle_identifier, is_kernel, application_id, upgrade_code) VALUES %s"+ds.dialect.OnConflictDoNothing("unique_identifier,source,extension_for"), titlesValues) titlesArgs := make([]any, 0, len(uniqueTitlesToInsert)*numberOfArgsPerSoftwareTitles) for _, title := range uniqueTitlesToInsert { @@ -1177,7 +1155,7 @@ func (ds *Datastore) preInsertSoftwareInventory( strings.Repeat("(?,?,?,?,?,?,?,?,?,?,?,?,?),", len(batchKeys)), ",", ) stmt := fmt.Sprintf( - `INSERT IGNORE INTO software ( + ds.dialect.InsertIgnoreInto()+` software ( name, version, source, @@ -1191,7 +1169,7 @@ func (ds *Datastore) preInsertSoftwareInventory( checksum, application_id, upgrade_code - ) VALUES %s`, + ) VALUES %s`+ds.dialect.OnConflictDoNothing("checksum"), values, ) @@ -1208,35 +1186,9 @@ func (ds *Datastore) preInsertSoftwareInventory( missingSoftwareTitles = append(missingSoftwareTitles, fmt.Sprintf("%s %s %s", sw.Name, sw.Version, sw.Source)) } - - // Use FMA canonical name if available, otherwise use osquery-reported name. - // This ensures software.name matches software_titles.name for consistency. - // - // IMPORTANT: The checksum is intentionally computed from osquery data - // (including the osquery-reported name, NOT the FMA name) for these reasons: - // - // 1. The checksum is used for deduplication via unique index. It serves as - // an internal identifier, not a content integrity hash. The stored name - // can differ from the name used in checksum computation. - // - // 2. Checksums are computed before FMA lookup, using raw osquery data. - // If we regenerated checksums with FMA names: - // - A cache miss or FMA sync delay could cause the same software to - // generate different checksums, creating duplicate entries. - // - Migration would require recomputing checksums for millions of rows. - // - // 3. The checksum is never recomputed from stored data - it's only computed - // from incoming osquery data during ingestion and used for lookup. - softwareName := sw.Name - if sw.BundleIdentifier != "" { - if fmaName, ok := fmaNames[sw.BundleIdentifier]; ok { - softwareName = fmaName - } - } - args = append( - args, softwareName, sw.Version, sw.Source, sw.Release, sw.Vendor, sw.Arch, - sw.BundleIdentifier, sw.ExtensionID, sw.ExtensionFor, titleID, checksum, sw.ApplicationID, sw.UpgradeCode, + args, sw.Name, sw.Version, sw.Source, sw.Release, sw.Vendor, sw.Arch, + sw.BundleIdentifier, sw.ExtensionID, sw.ExtensionFor, titleID, []byte(checksum), sw.ApplicationID, sw.UpgradeCode, ) } @@ -1276,9 +1228,9 @@ func (ds *Datastore) linkSoftwareToHost( var insertedSoftware []fleet.Software // Build map of all checksums we need to link - allChecksums := make([]string, 0, len(softwareChecksums)) + allChecksums := make([][]byte, 0, len(softwareChecksums)) for checksum := range softwareChecksums { - allChecksums = append(allChecksums, checksum) + allChecksums = append(allChecksums, []byte(checksum)) } // Get all software IDs (they should exist from pre-insertion). @@ -1292,7 +1244,7 @@ func (ds *Datastore) linkSoftwareToHost( // Build ID map softwareSummaryByChecksum := make(map[string]softwareSummary) for _, s := range allSoftwareSummaries { - softwareSummaryByChecksum[s.Checksum] = s + softwareSummaryByChecksum[string(s.Checksum)] = s } // Link software to host @@ -1315,7 +1267,7 @@ func (ds *Datastore) linkSoftwareToHost( // INSERT IGNORE handles duplicate key errors for idempotency. if len(insertsHostSoftware) > 0 { values := strings.TrimSuffix(strings.Repeat("(?,?,?),", len(insertsHostSoftware)/3), ",") - stmt := fmt.Sprintf(`INSERT IGNORE INTO host_software (host_id, software_id, last_opened_at) VALUES %s`, values) + stmt := fmt.Sprintf(ds.dialect.InsertIgnoreInto()+` host_software (host_id, software_id, last_opened_at) VALUES %s`+ds.dialect.OnConflictDoNothing("host_id,software_id"), values) if _, err := tx.ExecContext(ctx, stmt, insertsHostSoftware...); err != nil { return nil, ctxerr.Wrap(ctx, err, "insert host software") } @@ -1466,7 +1418,7 @@ func (ds *Datastore) reconcileExistingTitleEmptyWindowsUpgradeCodes( return nil } -func getExistingSoftwareSummariesByChecksums(ctx context.Context, tx sqlx.QueryerContext, checksums []string) ([]softwareSummary, error) { +func getExistingSoftwareSummariesByChecksums(ctx context.Context, tx sqlx.QueryerContext, checksums [][]byte) ([]softwareSummary, error) { if len(checksums) == 0 { return []softwareSummary{}, nil } @@ -1558,9 +1510,10 @@ func updateModifiedHostSoftwareDB( func updateSoftwareUpdatedAt( ctx context.Context, tx sqlx.ExtContext, + dialect DialectHelper, hostID uint, ) error { - const stmt = `INSERT INTO host_updates(host_id, software_updated_at) VALUES (?, CURRENT_TIMESTAMP) ON DUPLICATE KEY UPDATE software_updated_at=VALUES(software_updated_at)` + stmt := `INSERT INTO host_updates(host_id, software_updated_at) VALUES (?, CURRENT_TIMESTAMP) ` + dialect.OnDuplicateKey("host_id", "software_updated_at=VALUES(software_updated_at)") if _, err := tx.ExecContext(ctx, stmt, hostID); err != nil { return ctxerr.Wrap(ctx, err, "update host updates") @@ -1569,7 +1522,10 @@ func updateSoftwareUpdatedAt( return nil } -var dialect = goqu.Dialect("mysql") +// goquMySQLDialect is a package-level fallback for standalone functions that +// haven't been refactored to accept a goqu.DialectWrapper parameter yet. +// TODO(pg): remove once all standalone functions accept a dialect parameter. +var goquMySQLDialect = goqu.Dialect("mysql") // listSoftwareDB returns software installed on hosts. Use opts for pagination, filtering, and controlling // fields populated in the returned software. @@ -1866,7 +1822,7 @@ func selectSoftwareSQL(opts fleet.SoftwareListOptions) (string, []interface{}, e } // Fallback to the original goqu-based query builder for complex cases - ds := dialect. + ds := goquMySQLDialect. From(goqu.I("software").As("s")). Select( "s.id", @@ -2065,12 +2021,16 @@ func selectSoftwareSQL(opts fleet.SoftwareListOptions) (string, []interface{}, e "generated_cpe", ) + if opts.HostID != nil { + ds = ds.GroupByAppend("hs.last_opened_at") + } + // Pagination is a bit more complex here due to the join with software_cve table and aggregated columns from cve_meta table. // Apply order by again after joining on sub query ds = appendListOptionsToSelect(ds, opts.ListOptions) // join on software_cve and cve_meta after apply pagination using the sub-query above - ds = dialect.From(ds.As("s")). + ds = goquMySQLDialect.From(ds.As("s")). Select( "s.id", "s.name", @@ -2343,12 +2303,12 @@ func (ds *Datastore) AllSoftwareIterator( } if query.NameMatch != "" { - conditionals = append(conditionals, "s.name REGEXP ?") + conditionals = append(conditionals, ds.dialect.RegexpMatch("s.name", "?")) args = append(args, query.NameMatch) } if query.NameExclude != "" { - conditionals = append(conditionals, "s.name NOT REGEXP ?") + conditionals = append(conditionals, "NOT ("+ds.dialect.RegexpMatch("s.name", "?")+")") args = append(args, query.NameExclude) } @@ -2377,7 +2337,7 @@ func (ds *Datastore) UpsertSoftwareCPEs(ctx context.Context, cpes []fleet.Softwa values := strings.TrimSuffix(strings.Repeat("(?,?),", len(cpes)), ",") sql := fmt.Sprintf( - `INSERT INTO software_cpe (software_id, cpe) VALUES %s ON DUPLICATE KEY UPDATE cpe = VALUES(cpe)`, + `INSERT INTO software_cpe (software_id, cpe) VALUES %s `+ds.dialect.OnDuplicateKey("id", `cpe = VALUES(cpe)`), values, ) @@ -2523,9 +2483,11 @@ func (ds *Datastore) DeleteOutOfDateVulnerabilities(ctx context.Context, source func (ds *Datastore) DeleteOrphanedSoftwareVulnerabilities(ctx context.Context) error { if _, err := ds.writer(ctx).ExecContext(ctx, ` - DELETE sc FROM software_cve sc - LEFT JOIN host_software hs ON hs.software_id = sc.software_id - WHERE hs.host_id IS NULL + DELETE FROM software_cve + WHERE NOT EXISTS ( + SELECT 1 FROM host_software hs + WHERE hs.software_id = software_cve.software_id + ) `); err != nil { return ctxerr.Wrap(ctx, err, "deleting orphaned software vulnerabilities") } @@ -2533,7 +2495,7 @@ func (ds *Datastore) DeleteOrphanedSoftwareVulnerabilities(ctx context.Context) } func (ds *Datastore) SoftwareByID(ctx context.Context, id uint, teamID *uint, includeCVEScores bool, tmFilter *fleet.TeamFilter) (*fleet.Software, error) { - q := dialect.From(goqu.I("software").As("s")). + q := ds.dialect.GoquDialect().From(goqu.I("software").As("s")). Select( "s.id", "s.name", @@ -2563,7 +2525,7 @@ func (ds *Datastore) SoftwareByID(ctx context.Context, id uint, teamID *uint, in ) // join only on software_id as we'll need counts for all teams - // to filter down to the teams the user has access to + // to filter down to the team's the user has access to if tmFilter != nil { q = q.LeftJoin( goqu.I("software_host_counts").As("shc"), @@ -2663,26 +2625,6 @@ func (ds *Datastore) SoftwareByID(ctx context.Context, id uint, teamID *uint, in return &software, nil } -func (ds *Datastore) SoftwareLiteByID( - ctx context.Context, - id uint, -) (fleet.SoftwareLite, error) { - const stmt = ` - SELECT id, name, version - FROM software - WHERE id = ? - ` - var results fleet.SoftwareLite - if err := sqlx.GetContext(ctx, ds.reader(ctx), &results, stmt, id); err != nil { - if err == sql.ErrNoRows { - return fleet.SoftwareLite{}, notFound("Software").WithID(id) - } - return fleet.SoftwareLite{}, ctxerr.Wrap(ctx, err, "get software version name for host filter") - } - - return results, nil -} - // SyncHostsSoftware calculates the number of hosts having each // software installed and stores that information in the software_host_counts // table. @@ -2718,17 +2660,17 @@ func (ds *Datastore) SyncHostsSoftware(ctx context.Context, updatedAt time.Time) WHERE h.team_id IS NULL AND hs.software_id > ? AND hs.software_id <= ? GROUP BY hs.software_id` - insertStmt = ` + valuesPart = `(?, ?, ?, ?, ?),` + ) + + insertStmt := ` INSERT INTO ` + swapTable + ` (software_id, hosts_count, team_id, global_stats, updated_at) VALUES %s - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("host_id,software_id", ` hosts_count = VALUES(hosts_count), - updated_at = VALUES(updated_at)` - - valuesPart = `(?, ?, ?, ?, ?),` - ) + updated_at = VALUES(updated_at)`) // Create a fresh swap table to populate with new counts. If a previous run left a partial swap table, drop it first. w := ds.writer(ctx) @@ -2910,12 +2852,12 @@ func (ds *Datastore) CleanupSoftwareTitles(ctx context.Context) error { // Re-check orphan status on the writer to avoid deleting a title that an IT admin just linked // (e.g., added a software installer) between the reader SELECT and this DELETE. deleteOrphanedSoftwareTitlesStmt = ` - DELETE st FROM software_titles st - LEFT JOIN software s ON st.id = s.title_id - LEFT JOIN software_installers si ON st.id = si.title_id - LEFT JOIN in_house_apps iha ON st.id = iha.title_id - LEFT JOIN vpp_apps vap ON st.id = vap.title_id - WHERE st.id IN (?) AND s.title_id IS NULL AND si.title_id IS NULL AND iha.title_id IS NULL AND vap.title_id IS NULL` + DELETE FROM software_titles + WHERE id IN (?) + AND NOT EXISTS (SELECT 1 FROM software s WHERE s.title_id = software_titles.id) + AND NOT EXISTS (SELECT 1 FROM software_installers si WHERE si.title_id = software_titles.id) + AND NOT EXISTS (SELECT 1 FROM in_house_apps iha WHERE iha.title_id = software_titles.id) + AND NOT EXISTS (SELECT 1 FROM vpp_apps vap WHERE vap.title_id = software_titles.id)` ) var lastID uint @@ -3049,13 +2991,13 @@ func (ds *Datastore) InsertCVEMeta(ctx context.Context, cveMeta []fleet.CVEMeta) query := ` INSERT INTO cve_meta (cve, cvss_score, epss_probability, cisa_known_exploit, published, description) VALUES %s -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("cve", ` cvss_score = VALUES(cvss_score), epss_probability = VALUES(epss_probability), cisa_known_exploit = VALUES(cisa_known_exploit), published = VALUES(published), description = VALUES(description) -` +`) batchSize := 500 for i := 0; i < len(cveMeta); i += batchSize { @@ -3097,11 +3039,11 @@ func (ds *Datastore) InsertSoftwareVulnerability( stmt := ` INSERT INTO software_cve (cve, source, software_id, resolved_in_version) VALUES (?,?,?,?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("id", ` source = VALUES(source), resolved_in_version = VALUES(resolved_in_version), updated_at=? - ` + `) args = append(args, vuln.CVE, source, vuln.SoftwareID, vuln.ResolvedInVersion, time.Now().UTC()) res, err := ds.writer(ctx).ExecContext(ctx, stmt, args...) @@ -3174,11 +3116,11 @@ func (ds *Datastore) InsertSoftwareVulnerabilities( stmt := fmt.Sprintf(` INSERT INTO software_cve (cve, source, software_id, resolved_in_version) VALUES %s - ON DUPLICATE KEY UPDATE + `+ds.dialect.OnDuplicateKey("id", ` source = VALUES(source), resolved_in_version = VALUES(resolved_in_version), updated_at = ? - `, values) + `), values) var args []any for _, v := range batch { @@ -3210,7 +3152,7 @@ func (ds *Datastore) ListSoftwareVulnerabilitiesByHostIDsSource( } var queryR []softwareVulnerabilityWithHostId - stmt := dialect. + stmt := ds.dialect.GoquDialect(). From(goqu.T("software_cve").As("sc")). Join( goqu.T("host_software").As("hs"), @@ -3314,7 +3256,7 @@ func (ds *Datastore) ListCVEs(ctx context.Context, maxAge time.Duration) ([]flee var result []fleet.CVEMeta maxAgeDate := time.Now().Add(-1 * maxAge) - stmt := dialect.From(goqu.T("cve_meta")). + stmt := ds.dialect.GoquDialect().From(goqu.T("cve_meta")). Select( goqu.C("cve"), goqu.C("cvss_score"), @@ -5837,6 +5779,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt } var replacements []any + gc := ds.dialect.GroupConcat if len(softwareTitleIDs) > 0 { replacements = append(replacements, // For software installers @@ -5852,12 +5795,12 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt software_installers.filename AS package_name, software_installers.version AS package_version, software_installers.platform as package_platform, - GROUP_CONCAT(software.id) AS software_id_list, - GROUP_CONCAT(software.source) AS software_source_list, - GROUP_CONCAT(software.extension_for) AS software_extension_for_list, - GROUP_CONCAT(software.upgrade_code) AS software_upgrade_code_list, - GROUP_CONCAT(software.version) AS version_list, - GROUP_CONCAT(software.bundle_identifier) AS bundle_identifier_list, + `+gc("software.id", ",")+` AS software_id_list, + `+gc("software.source", ",")+` AS software_source_list, + `+gc("software.extension_for", ",")+` AS software_extension_for_list, + `+gc("software.upgrade_code", ",")+` AS software_upgrade_code_list, + `+gc("software.version", ",")+` AS version_list, + `+gc("software.bundle_identifier", ",")+` AS bundle_identifier_list, NULL AS vpp_app_adam_id_list, NULL AS vpp_app_version_list, NULL AS vpp_app_platform_list, @@ -5903,11 +5846,11 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt NULL AS software_upgrade_code_list, NULL AS version_list, NULL AS bundle_identifier_list, - GROUP_CONCAT(vpp_apps.adam_id) AS vpp_app_adam_id_list, - GROUP_CONCAT(vpp_apps.latest_version) AS vpp_app_version_list, - GROUP_CONCAT(vpp_apps.platform) as vpp_app_platform_list, - GROUP_CONCAT(vpp_apps.icon_url) AS vpp_app_icon_url_list, - GROUP_CONCAT(vpp_apps_teams.self_service) AS vpp_app_self_service_list, + `+gc("vpp_apps.adam_id", ",")+` AS vpp_app_adam_id_list, + `+gc("vpp_apps.latest_version", ",")+` AS vpp_app_version_list, + `+gc("vpp_apps.platform", ",")+` as vpp_app_platform_list, + `+gc("vpp_apps.icon_url", ",")+` AS vpp_app_icon_url_list, + `+gc("vpp_apps_teams.self_service", ",")+` AS vpp_app_self_service_list, NULL AS in_house_app_id_list, NULL AS in_house_app_name_list, NULL AS in_house_app_version_list, @@ -5949,11 +5892,11 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt NULL as vpp_app_platform_list, NULL AS vpp_app_icon_url_list, NULL AS vpp_app_self_service_list, - GROUP_CONCAT(in_house_apps.id) AS in_house_app_id_list, - GROUP_CONCAT(in_house_apps.filename) AS in_house_app_name_list, - GROUP_CONCAT(in_house_apps.version) AS in_house_app_version_list, - GROUP_CONCAT(in_house_apps.platform) as in_house_app_platform_list, - GROUP_CONCAT(in_house_apps.self_service) as in_house_app_self_service_list + `+gc("in_house_apps.id", ",")+` AS in_house_app_id_list, + `+gc("in_house_apps.filename", ",")+` AS in_house_app_name_list, + `+gc("in_house_apps.version", ",")+` AS in_house_app_version_list, + `+gc("in_house_apps.platform", ",")+` as in_house_app_platform_list, + `+gc("in_house_apps.self_service", ",")+` as in_house_app_self_service_list `, ` GROUP BY software_titles.id, @@ -6484,7 +6427,7 @@ func (ds *Datastore) CreateIntermediateInstallFailureRecord(ctx context.Context, // Create or update a record with the failure details // Use INSERT ... ON DUPLICATE KEY UPDATE to make this idempotent - const insertStmt = ` + insertStmt := ` INSERT INTO host_software_installs ( execution_id, host_id, @@ -6502,14 +6445,14 @@ func (ds *Datastore) CreateIntermediateInstallFailureRecord(ctx context.Context, post_install_script_exit_code, post_install_script_output ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON DUPLICATE KEY UPDATE + ` + ds.dialect.OnDuplicateKey("id", ` install_script_exit_code = VALUES(install_script_exit_code), install_script_output = VALUES(install_script_output), pre_install_query_output = VALUES(pre_install_query_output), post_install_script_exit_code = VALUES(post_install_script_exit_code), post_install_script_output = VALUES(post_install_script_output), updated_at = CURRENT_TIMESTAMP(6) - ` + `) truncateOutput := func(output *string) *string { if output != nil { @@ -6637,12 +6580,11 @@ WHERE hvsi.host_id = ? AND st.id IN (?) func (ds *Datastore) NewSoftwareCategory(ctx context.Context, name string) (*fleet.SoftwareCategory, error) { stmt := `INSERT INTO software_categories (name) VALUES (?)` - res, err := ds.writer(ctx).ExecContext(ctx, stmt, name) + r, err := ds.insertAndGetID(ctx, ds.writer(ctx), stmt, name) if err != nil { return nil, ctxerr.Wrap(ctx, err, "new software category") } - r, _ := res.LastInsertId() id := uint(r) //nolint:gosec // dismiss G115 return &fleet.SoftwareCategory{Name: name, ID: id}, nil } @@ -6763,3 +6705,23 @@ WHERE return ret, nil } +func (ds *Datastore) SoftwareLiteByID( + ctx context.Context, + id uint, +) (fleet.SoftwareLite, error) { + const stmt = ` + SELECT id, name, version + FROM software + WHERE id = ? + ` + var results fleet.SoftwareLite + if err := sqlx.GetContext(ctx, ds.reader(ctx), &results, stmt, id); err != nil { + if err == sql.ErrNoRows { + return fleet.SoftwareLite{}, notFound("Software").WithID(id) + } + return fleet.SoftwareLite{}, ctxerr.Wrap(ctx, err, "get software version name for host filter") + } + + return results, nil +} + diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index f8f7535c64c..b1b65091597 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -94,7 +94,7 @@ func (ds *Datastore) GetSoftwareInstallDetails(ctx context.Context, executionId ua.host_id AS host_id, ua.execution_id AS execution_id, siua.software_installer_id AS installer_id, - ua.payload->'$.self_service' AS self_service, + COALESCE(ua.payload->>'$.self_service', '0') = '1' AS self_service, COALESCE(si.pre_install_query, '') AS pre_install_condition, inst.contents AS install_script, uninst.contents AS uninstall_script, @@ -324,7 +324,7 @@ INSERT INTO software_installers ( url, upgrade_code, is_active, - patch_query + patch_query ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, (SELECT name FROM users WHERE id = ?), (SELECT email FROM users WHERE id = ?), ?, ?, ?, ?, ?)` args := []interface{}{ @@ -352,9 +352,9 @@ INSERT INTO software_installers ( payload.PatchQuery, } - res, err := tx.ExecContext(ctx, stmt, args...) + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, stmt, args...) if err != nil { - if IsDuplicate(err) { + if ds.dialect.IsDuplicate(err) { // already exists for this team/no team teamName, err := ds.getTeamName(ctx, payload.TeamID) if err != nil { @@ -365,10 +365,9 @@ INSERT INTO software_installers ( return err } - id, _ := res.LastInsertId() installerID = uint(id) //nolint:gosec // dismiss G115 - if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, installerID, *payload.ValidatedLabels, softwareTypeInstaller); err != nil { + if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, ds.dialect, installerID, *payload.ValidatedLabels, softwareTypeInstaller); err != nil { return ctxerr.Wrap(ctx, err, "upsert software installer labels") } @@ -487,7 +486,7 @@ func (ds *Datastore) createAutomaticPolicy(ctx context.Context, tx sqlx.ExtConte SoftwareInstallerID: softwareInstallerID, VPPAppsTeamsID: vppAppsTeamsID, Type: fleet.PolicyTypeDynamic, - }) + }, ds.dialect) if err != nil { return nil, ctxerr.Wrap(ctx, err, "create automatic policy query") } @@ -553,14 +552,6 @@ func (ds *Datastore) getOrGenerateSoftwareInstallerTitleID(ctx context.Context, if payload.Source == "programs" && payload.UpgradeCode != "" { updateStmt := `UPDATE software_titles SET upgrade_code = ? WHERE id = ?` updateArgs := []any{payload.UpgradeCode, titleID} - - // Update the software title name if this is a Windows FMA with an upgrade code. We already update - // software titles with macOS FMA names on FMA catalog sync, so we only do Windows here. - if payload.FleetMaintainedAppID != nil { - updateStmt = `UPDATE software_titles SET name = ?, upgrade_code = ? WHERE id = ?` - updateArgs = []any{payload.Title, payload.UpgradeCode, titleID} - } - _, err := ds.writer(ctx).ExecContext(ctx, updateStmt, updateArgs...) if err != nil { return 0, err @@ -588,7 +579,7 @@ func (ds *Datastore) addSoftwareTitleToMatchingSoftware(ctx context.Context, tit args = append(args, whereArgs...) updateSoftwareStmt := fmt.Sprintf(` UPDATE software s - SET s.title_id = ? + SET title_id = ? %s`, whereClause) _, err := ds.writer(ctx).ExecContext(ctx, updateSoftwareStmt, args...) return ctxerr.Wrap(ctx, err, "adding fk reference in software to software_titles") @@ -604,7 +595,7 @@ const ( // setOrUpdateSoftwareInstallerLabelsDB sets or updates the label associations for the specified software // installer. If no labels are provided, it will remove all label associations with the software installer. -func setOrUpdateSoftwareInstallerLabelsDB(ctx context.Context, tx sqlx.ExtContext, installerID uint, labels fleet.LabelIdentsWithScope, softwareType softwareType) error { +func setOrUpdateSoftwareInstallerLabelsDB(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, installerID uint, labels fleet.LabelIdentsWithScope, softwareType softwareType) error { labelIds := make([]uint, 0, len(labels.ByName)) for _, label := range labels.ByName { labelIds = append(labelIds, label.LabelID) @@ -645,7 +636,7 @@ func setOrUpdateSoftwareInstallerLabelsDB(ctx context.Context, tx sqlx.ExtContex } stmt := `INSERT INTO %[1]s_labels (%[1]s_id, label_id, exclude, require_all) VALUES %s - ON DUPLICATE KEY UPDATE exclude = VALUES(exclude), require_all = VALUES(require_all)` + ` + dialect.OnDuplicateKey("id", "exclude = VALUES(exclude), require_all = VALUES(require_all)") var placeholders string var insertArgs []interface{} for _, lid := range labelIds { @@ -741,7 +732,7 @@ func (ds *Datastore) SaveInstallerUpdates(ctx context.Context, payload *fleet.Up } if payload.ValidatedLabels != nil { - if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, payload.InstallerID, *payload.ValidatedLabels, softwareTypeInstaller); err != nil { + if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, ds.dialect, payload.InstallerID, *payload.ValidatedLabels, softwareTypeInstaller); err != nil { return ctxerr.Wrap(ctx, err, "upsert software installer labels") } } @@ -753,7 +744,7 @@ func (ds *Datastore) SaveInstallerUpdates(ctx context.Context, payload *fleet.Up } if payload.DisplayName != nil { - if err := updateSoftwareTitleDisplayName(ctx, tx, payload.TeamID, payload.TitleID, *payload.DisplayName); err != nil { + if err := updateSoftwareTitleDisplayName(ctx, tx, ds.dialect, payload.TeamID, payload.TitleID, *payload.DisplayName); err != nil { return ctxerr.Wrap(ctx, err, "update software title display name") } } @@ -890,7 +881,8 @@ SELECT COALESCE(st.name, '') AS software_title, si.platform, si.fleet_maintained_app_id, - si.upgrade_code + si.upgrade_code, + si.patch_query FROM software_installers si LEFT OUTER JOIN software_titles st ON st.id = si.title_id @@ -1172,7 +1164,7 @@ func (ds *Datastore) DeleteSoftwareInstaller(ctx context.Context, id uint) error // allow delete only if install_during_setup is false res, err := tx.ExecContext(ctx, `DELETE FROM software_installers WHERE id = ? AND install_during_setup = 0`, id) if err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { // Check if the software installer is referenced by a policy automation. var count int if err := sqlx.GetContext(ctx, tx, &count, `SELECT COUNT(*) FROM policies WHERE software_installer_id = ?`, id); err != nil { @@ -1294,12 +1286,12 @@ INSERT INTO upcoming_activities VALUES (?, ?, ?, ?, 'software_install', ?, JSON_OBJECT( - 'self_service', ?, + 'self_service', CAST(? AS UNSIGNED), 'installer_filename', ?, 'version', ?, 'software_title_name', ?, 'source', ?, - 'with_retries', ?, + 'with_retries', CAST(? AS UNSIGNED), 'user', (SELECT JSON_OBJECT('name', name, 'email', email, 'gravatar_url', gravatar_url) FROM users WHERE id = ?) ) )` @@ -1347,26 +1339,33 @@ VALUES } execID := uuid.NewString() + // Convert booleans to int for JSON_OBJECT compatibility with PG's jsonb_build_object. + selfServiceInt := 0 + if opts.SelfService { + selfServiceInt = 1 + } + withRetriesInt := 0 + if opts.WithRetries { + withRetriesInt = 1 + } err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - res, err := tx.ExecContext(ctx, insertUAStmt, + activityID, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertUAStmt, hostID, opts.Priority(), userID, opts.IsFleetInitiated(), execID, - opts.SelfService, + selfServiceInt, installerDetails.Filename, installerDetails.Version, installerDetails.TitleName, installerDetails.Source, - opts.WithRetries, + withRetriesInt, userID, ) if err != nil { return ctxerr.Wrap(ctx, err, "insert software install request") } - - activityID, _ := res.LastInsertId() _, err = tx.ExecContext(ctx, insertSIUAStmt, activityID, softwareInstallerID, @@ -1485,7 +1484,7 @@ VALUES 'software_title_name', ?, 'source', ?, 'user', (SELECT JSON_OBJECT('name', name, 'email', email, 'gravatar_url', gravatar_url) FROM users WHERE id = ?), - 'self_service', ? + 'self_service', CAST(? AS UNSIGNED) ) )` @@ -1526,8 +1525,12 @@ VALUES userID = &ctxUser.ID } + selfServiceInt := 0 + if selfService { + selfServiceInt = 1 + } err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - res, err := tx.ExecContext(ctx, insertUAStmt, + activityID, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertUAStmt, hostID, 0, // Uninstalls are never used in setup experience, so always default priority userID, @@ -1536,13 +1539,11 @@ VALUES installerDetails.TitleName, installerDetails.Source, userID, - selfService, + selfServiceInt, ) if err != nil { return err } - - activityID, _ := res.LastInsertId() _, err = tx.ExecContext(ctx, insertSIUAStmt, activityID, softwareInstallerID, @@ -1608,7 +1609,7 @@ SELECT ua.user_id AS user_id, NULL AS post_install_script_exit_code, NULL AS install_script_exit_code, - ua.payload->'$.self_service' AS self_service, + COALESCE(ua.payload->>'$.self_service', '0') = '1' AS self_service, NULL AS host_deleted_at, siua.policy_id AS policy_id, ua.created_at as created_at, @@ -2074,17 +2075,17 @@ func (ds *Datastore) CleanupUnusedSoftwareInstallers(ctx context.Context, softwa const maxCachedFMAVersions = 2 func (ds *Datastore) BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { - const upsertSoftwareTitles = ` + upsertSoftwareTitles := ` INSERT INTO software_titles (name, source, extension_for, bundle_identifier, upgrade_code) VALUES %s -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("id", ` name = VALUES(name), source = VALUES(source), extension_for = VALUES(extension_for), bundle_identifier = VALUES(bundle_identifier) -` +`) const loadSoftwareTitles = ` SELECT @@ -2291,7 +2292,7 @@ WHERE title_id = ? ` - const insertNewOrEditedInstaller = ` + insertNewOrEditedInstaller := ` INSERT INTO software_installers ( team_id, global_or_team_id, @@ -2320,7 +2321,7 @@ INSERT INTO software_installers ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, (SELECT name FROM users WHERE id = ?), (SELECT email FROM users WHERE id = ?), ?, ?, COALESCE(?, false), ?, ?, ? ) -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("id", ` install_script_content_id = VALUES(install_script_content_id), uninstall_script_content_id = VALUES(uninstall_script_content_id), post_install_script_content_id = VALUES(post_install_script_content_id), @@ -2336,10 +2337,10 @@ ON DUPLICATE KEY UPDATE user_name = VALUES(user_name), user_email = VALUES(user_email), url = VALUES(url), + patch_query = VALUES(patch_query), install_during_setup = COALESCE(?, install_during_setup), - is_active = VALUES(is_active), - patch_query = VALUES(patch_query) -` + is_active = VALUES(is_active) +`) const updateInstaller = ` UPDATE @@ -2382,7 +2383,7 @@ WHERE software_installer_id = ? ` - const upsertInstallerLabels = ` + upsertInstallerLabels := ` INSERT INTO software_installer_labels ( software_installer_id, @@ -2392,10 +2393,10 @@ INSERT INTO ) VALUES %s -ON DUPLICATE KEY UPDATE +` + ds.dialect.OnDuplicateKey("id", ` exclude = VALUES(exclude), require_all = VALUES(require_all) -` +`) const loadExistingInstallerLabels = ` SELECT @@ -2423,8 +2424,7 @@ WHERE software_category_id NOT IN (?) ` - const upsertInstallerCategories = ` -INSERT IGNORE INTO + const upsertInstallerCategoriesSuffix = ` software_installer_software_categories ( software_installer_id, software_category_id @@ -2686,26 +2686,23 @@ WHERE return ctxerr.Errorf(ctx, "labels have not been validated for installer with name %s", installer.Filename) } - isRes, err := insertScriptContents(ctx, tx, installer.InstallScript) + installScriptID, err := insertScriptContents(ctx, tx, ds.dialect, installer.InstallScript) if err != nil { return ctxerr.Wrapf(ctx, err, "inserting install script contents for software installer with name %q", installer.Filename) } - installScriptID, _ := isRes.LastInsertId() - uisRes, err := insertScriptContents(ctx, tx, installer.UninstallScript) + uninstallScriptID, err := insertScriptContents(ctx, tx, ds.dialect, installer.UninstallScript) if err != nil { return ctxerr.Wrapf(ctx, err, "inserting uninstall script contents for software installer with name %q", installer.Filename) } - uninstallScriptID, _ := uisRes.LastInsertId() var postInstallScriptID *int64 if installer.PostInstallScript != "" { - pisRes, err := insertScriptContents(ctx, tx, installer.PostInstallScript) + insertID, err := insertScriptContents(ctx, tx, ds.dialect, installer.PostInstallScript) if err != nil { return ctxerr.Wrapf(ctx, err, "inserting post-install script contents for software installer with name %q", installer.Filename) } - insertID, _ := pisRes.LastInsertId() postInstallScriptID = &insertID } @@ -3038,7 +3035,7 @@ WHERE upsertCategoriesArgs = append(upsertCategoriesArgs, installerID, catID) } upsertCategoriesValues := strings.TrimSuffix(strings.Repeat("(?,?),", len(installer.CategoryIDs)), ",") - _, err = tx.ExecContext(ctx, fmt.Sprintf(upsertInstallerCategories, upsertCategoriesValues), upsertCategoriesArgs...) + _, err = tx.ExecContext(ctx, ds.dialect.InsertIgnoreInto()+fmt.Sprintf(upsertInstallerCategoriesSuffix, upsertCategoriesValues)+ds.dialect.OnConflictDoNothing("software_installer_id,software_category_id"), upsertCategoriesArgs...) if err != nil { return ctxerr.Wrapf(ctx, err, "insert new/edited categories for installer with name %q", installer.Filename) } @@ -3047,7 +3044,7 @@ WHERE // update display name for the software title if it needs to be updated or inserted // no deletions will happen, display names will be set to empty if needed if name, ok := displayNameIDMap[titleID]; (ok && name != installer.DisplayName) || (!ok && installer.DisplayName != "") { - if err := updateSoftwareTitleDisplayName(ctx, tx, tmID, titleID, installer.DisplayName); err != nil { + if err := updateSoftwareTitleDisplayName(ctx, tx, ds.dialect, tmID, titleID, installer.DisplayName); err != nil { return ctxerr.Wrapf(ctx, err, "update software title display name for installer with name %q", installer.Filename) } } @@ -3115,7 +3112,7 @@ func (ds *Datastore) GetDetailsForUninstallFromExecutionID(ctx context.Context, UNION - SELECT st.name, COALESCE(ua.payload->'$.self_service', FALSE) self_service + SELECT st.name, COALESCE(ua.payload->>'$.self_service', 'false') self_service FROM software_titles st INNER JOIN software_installers si ON si.title_id = st.id diff --git a/server/datastore/mysql/software_title_display_names.go b/server/datastore/mysql/software_title_display_names.go index 4e5f7d13d53..33311f3e4ae 100644 --- a/server/datastore/mysql/software_title_display_names.go +++ b/server/datastore/mysql/software_title_display_names.go @@ -8,7 +8,7 @@ import ( "github.com/jmoiron/sqlx" ) -func updateSoftwareTitleDisplayName(ctx context.Context, tx sqlx.ExtContext, teamID *uint, titleID uint, displayName string) error { +func updateSoftwareTitleDisplayName(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, teamID *uint, titleID uint, displayName string) error { var tmID uint if teamID != nil { tmID = *teamID @@ -17,8 +17,7 @@ func updateSoftwareTitleDisplayName(ctx context.Context, tx sqlx.ExtContext, tea INSERT INTO software_title_display_names (team_id, software_title_id, display_name) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE - display_name = VALUES(display_name)`, tmID, titleID, displayName) + `+dialect.OnDuplicateKey("title_id", "display_name = VALUES(display_name)"), tmID, titleID, displayName) if err != nil { return err } diff --git a/server/datastore/mysql/software_title_icons.go b/server/datastore/mysql/software_title_icons.go index d0ae9c1e1d9..1b53c9af359 100644 --- a/server/datastore/mysql/software_title_icons.go +++ b/server/datastore/mysql/software_title_icons.go @@ -15,8 +15,7 @@ func (ds *Datastore) CreateOrUpdateSoftwareTitleIcon(ctx context.Context, payloa var args []any query = ` INSERT INTO software_title_icons (team_id, software_title_id, storage_id, filename) - VALUES (?, ?, ?, ?) ON DUPLICATE KEY UPDATE - storage_id = VALUES(storage_id), filename = VALUES(filename) + VALUES (?, ?, ?, ?) ` + ds.dialect.OnDuplicateKey("id", `storage_id = VALUES(storage_id), filename = VALUES(filename)`) + ` ` args = []any{payload.TeamID, payload.TitleID, payload.StorageID, payload.Filename} diff --git a/server/datastore/mysql/software_titles.go b/server/datastore/mysql/software_titles.go index 25b4ada8baf..5c176529da0 100644 --- a/server/datastore/mysql/software_titles.go +++ b/server/datastore/mysql/software_titles.go @@ -1082,7 +1082,7 @@ SELECT s.title_id, s.id, s.version, %s -- placeholder for optional host_counts - CONCAT('[', GROUP_CONCAT(JSON_QUOTE(scve.cve) SEPARATOR ','), ']') as vulnerabilities + CONCAT('[', ` + ds.dialect.GroupConcat("JSON_QUOTE(scve.cve)", ",") + `, ']') as vulnerabilities FROM software s LEFT JOIN software_host_counts shc ON shc.software_id = s.id AND %s LEFT JOIN software_cve scve ON shc.software_id = scve.software_id @@ -1159,17 +1159,16 @@ func (ds *Datastore) SyncHostsSoftwareTitles(ctx context.Context, updatedAt time WHERE h.team_id IS NULL AND hs.software_id > 0 GROUP BY st.id` - insertStmt = ` + valuesPart = `(?, ?, ?, ?, ?),` + ) + + insertStmt := ` INSERT INTO ` + swapTable + ` (software_title_id, hosts_count, team_id, global_stats, updated_at) VALUES %s - ON DUPLICATE KEY UPDATE - hosts_count = VALUES(hosts_count), - updated_at = VALUES(updated_at)` - - valuesPart = `(?, ?, ?, ?, ?),` - ) + ` + ds.dialect.OnDuplicateKey("software_id,team_id", `hosts_count = VALUES(hosts_count), + updated_at = VALUES(updated_at)`) // Create a fresh swap table to populate with new counts. If a previous run left a partial swap table, drop it first. w := ds.writer(ctx) @@ -1280,11 +1279,9 @@ func (ds *Datastore) UpdateSoftwareTitleAutoUpdateConfig(ctx context.Context, ti INSERT INTO software_update_schedules (title_id, team_id, enabled, start_time, end_time) VALUES (?, ?, ?, ?, ?) -ON DUPLICATE KEY UPDATE - enabled = VALUES(enabled), +` + ds.dialect.OnDuplicateKey("id", `enabled = VALUES(enabled), start_time = IF(VALUES(start_time) = '', start_time, VALUES(start_time)), - end_time = IF(VALUES(end_time) = '', end_time, VALUES(end_time)) -` + end_time = IF(VALUES(end_time) = '', end_time, VALUES(end_time))`) _, err := ds.writer(ctx).ExecContext(ctx, stmt, titleID, teamID, config.AutoUpdateEnabled, startTime, endTime) if err != nil { return ctxerr.Wrap(ctx, err, "updating software title auto update config") diff --git a/server/datastore/mysql/statistics.go b/server/datastore/mysql/statistics.go index 45629931751..71806cfba74 100644 --- a/server/datastore/mysql/statistics.go +++ b/server/datastore/mysql/statistics.go @@ -3,6 +3,7 @@ package mysql import ( "context" "database/sql" + "fmt" "time" "github.com/fleetdm/fleet/v4/server" @@ -270,7 +271,7 @@ func (ds *Datastore) getTableRowCountsViaInformationSchema(ctx context.Context) ctx, ds.reader(ctx), &results, - "SELECT table_name, COALESCE(table_rows, 0) table_rows FROM information_schema.tables WHERE table_schema = (SELECT DATABASE())", + fmt.Sprintf("SELECT table_name, COALESCE(table_rows, 0) table_rows FROM information_schema.tables WHERE table_schema = %s", ds.currentDatabaseFn()), ); err != nil { return nil, err } diff --git a/server/datastore/mysql/teams.go b/server/datastore/mysql/teams.go index 59ac6abb1d1..5a67bf7cc1c 100644 --- a/server/datastore/mysql/teams.go +++ b/server/datastore/mysql/teams.go @@ -33,9 +33,7 @@ func (ds *Datastore) NewTeam(ctx context.Context, team *fleet.Team) (*fleet.Team config ) VALUES (?, ?, ?, ?) ` - result, err := tx.ExecContext( - ctx, - query, + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, query, team.Name, team.Filename, team.Description, @@ -45,7 +43,6 @@ func (ds *Datastore) NewTeam(ctx context.Context, team *fleet.Team) (*fleet.Team return ctxerr.Wrap(ctx, err, "insert team") } - id, _ := result.LastInsertId() team.ID = uint(id) //nolint:gosec // dismiss G115 team.CreatedAt = time.Now().UTC().Truncate(time.Second) @@ -134,6 +131,7 @@ var teamRefs = []string{ "mdm_windows_configuration_profiles", "mdm_apple_declarations", "mdm_android_configuration_profiles", + "android_app_configurations", "certificate_templates", "software_title_icons", "software_title_display_names", @@ -152,13 +150,6 @@ var teamLabelsRefs = []string{ } func (ds *Datastore) DeleteTeam(ctx context.Context, tid uint) error { - // Enqueue commands for Windows profiles. This must run - // first because the main transaction deletes the config profile rows - // (which contain the SyncML bytes needed to generate commands). - if err := ds.enqueueWindowsDeleteCommandsForTeam(ctx, tid); err != nil { - return ctxerr.Wrapf(ctx, err, "enqueuing windows delete commands for team %d", tid) - } - return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { // Delete team policies first, because policies can have associated installers and scripts // which may be deleted on cascade before deleting the policies (which are also deleted on cascade). @@ -215,34 +206,6 @@ func (ds *Datastore) DeleteTeam(ctx context.Context, tid uint) error { }) } -// enqueueWindowsDeleteCommandsForTeam generates SyncML commands for -// Windows profiles assigned to the given team. Runs in its own transaction to -// keep load out of the main DeleteTeam transaction. -func (ds *Datastore) enqueueWindowsDeleteCommandsForTeam(ctx context.Context, tid uint) error { - return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - var profRows []struct { - ProfileUUID string `db:"profile_uuid"` - SyncML []byte `db:"syncml"` - } - if err := sqlx.SelectContext(ctx, tx, &profRows, - `SELECT profile_uuid, syncml FROM mdm_windows_configuration_profiles WHERE team_id = ?`, tid); err != nil { - return ctxerr.Wrapf(ctx, err, "loading windows profiles for team %d", tid) - } - if len(profRows) == 0 { - return nil - } - - profileUUIDs := make([]string, 0, len(profRows)) - profileContents := make(map[string][]byte, len(profRows)) - for _, r := range profRows { - profileUUIDs = append(profileUUIDs, r.ProfileUUID) - profileContents[r.ProfileUUID] = r.SyncML - } - - return ds.cancelWindowsHostInstallsForDeletedMDMProfiles(ctx, tx, profileUUIDs, profileContents) - }) -} - func (ds *Datastore) TeamByName(ctx context.Context, name string) (*fleet.Team, error) { // We must normalize the name for full Unicode support (Unicode equivalence). nameUnicode := norm.NFC.String(name) @@ -653,7 +616,7 @@ func (ds *Datastore) SaveDefaultTeamConfig(ctx context.Context, config *fleet.Te _, err = ds.writer(ctx).ExecContext(ctx, `INSERT INTO default_team_config_json(id, json_value) VALUES(1, ?) - ON DUPLICATE KEY UPDATE json_value = VALUES(json_value)`, + `+ds.dialect.OnDuplicateKey("id", `json_value = VALUES(json_value)`), configBytes, ) return ctxerr.Wrap(ctx, err, "save default team config") diff --git a/server/datastore/mysql/users.go b/server/datastore/mysql/users.go index 4e4b4938528..5b27b433b78 100644 --- a/server/datastore/mysql/users.go +++ b/server/datastore/mysql/users.go @@ -53,7 +53,7 @@ func (ds *Datastore) NewUser(ctx context.Context, user *fleet.User) (*fleet.User invite_id ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?) ` - result, err := tx.ExecContext(ctx, sqlStatement, + id, err := insertAndGetIDTx(ctx, tx, ds.dialect, sqlStatement, user.Password, user.Salt, user.Name, @@ -76,7 +76,6 @@ func (ds *Datastore) NewUser(ctx context.Context, user *fleet.User) (*fleet.User return ctxerr.Wrap(ctx, err, "create new user") } - id, _ := result.LastInsertId() user.ID = uint(id) //nolint:gosec // dismiss G115 if err := saveTeamsForUserDB(ctx, tx, user); err != nil { @@ -385,9 +384,8 @@ func (ds *Datastore) DeleteUser(ctx context.Context, id uint) error { SELECT u.id, u.name, u.email FROM users AS u WHERE u.id = ? - ON DUPLICATE KEY UPDATE - name = u.name, - email = u.email` + ` + ds.dialect.OnDuplicateKey("id", `name = VALUES(name), + email = VALUES(email)`) _, err := ds.writer(ctx).ExecContext(ctx, stmt, id) if err != nil { return ctxerr.Wrap(ctx, err, "populate users_deleted entry") diff --git a/server/datastore/mysql/vpp.go b/server/datastore/mysql/vpp.go index 982630e2b78..6f0c22e9cd4 100644 --- a/server/datastore/mysql/vpp.go +++ b/server/datastore/mysql/vpp.go @@ -341,7 +341,7 @@ func (ds *Datastore) BatchInsertVPPApps(ctx context.Context, apps []*fleet.VPPAp app.TitleID = titleID - if err := insertVPPApps(ctx, tx, []*fleet.VPPApp{app}); err != nil { + if err := insertVPPApps(ctx, tx, ds.dialect, []*fleet.VPPApp{app}); err != nil { return ctxerr.Wrap(ctx, err, "BatchInsertVPPApps insertVPPApps transaction") } } @@ -508,7 +508,7 @@ func (ds *Datastore) SetTeamVPPApps(ctx context.Context, teamID *uint, incomingA if vppToken != nil { tokenID = &vppToken.ID } - vppAppTeamID, err := insertVPPAppTeams(ctx, tx, toAdd, teamID, tokenID) + vppAppTeamID, err := insertVPPAppTeams(ctx, tx, ds.dialect, toAdd, teamID, tokenID) if err != nil { return ctxerr.Wrap(ctx, err, "SetTeamVPPApps inserting vpp app into team") } @@ -522,7 +522,7 @@ func (ds *Datastore) SetTeamVPPApps(ctx context.Context, teamID *uint, incomingA } if toAdd.ValidatedLabels != nil { - if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, vppAppTeamID, *toAdd.ValidatedLabels, softwareTypeVPP); err != nil { + if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, ds.dialect, vppAppTeamID, *toAdd.ValidatedLabels, softwareTypeVPP); err != nil { return ctxerr.Wrap(ctx, err, "failed to update labels on vpp apps batch operation") } } @@ -534,7 +534,7 @@ func (ds *Datastore) SetTeamVPPApps(ctx context.Context, teamID *uint, incomingA } if toAdd.DisplayName != nil { - if err := updateSoftwareTitleDisplayName(ctx, tx, teamID, appStoreAppIDsToTitleIDs[toAdd.VPPAppID.String()], *toAdd.DisplayName); err != nil { + if err := updateSoftwareTitleDisplayName(ctx, tx, ds.dialect, teamID, appStoreAppIDsToTitleIDs[toAdd.VPPAppID.String()], *toAdd.DisplayName); err != nil { return ctxerr.Wrap(ctx, err, "setting software title display name for vpp app") } } @@ -639,11 +639,11 @@ func (ds *Datastore) InsertVPPAppWithTeam(ctx context.Context, app *fleet.VPPApp app.TitleID = titleID - if err := insertVPPApps(ctx, tx, []*fleet.VPPApp{app}); err != nil { + if err := insertVPPApps(ctx, tx, ds.dialect, []*fleet.VPPApp{app}); err != nil { return ctxerr.Wrap(ctx, err, "InsertVPPAppWithTeam insertVPPApps transaction") } - vppAppTeamID, err := insertVPPAppTeams(ctx, tx, app.VPPAppTeam, teamID, vppTokenID) + vppAppTeamID, err := insertVPPAppTeams(ctx, tx, ds.dialect, app.VPPAppTeam, teamID, vppTokenID) if err != nil { return ctxerr.Wrap(ctx, err, "InsertVPPAppWithTeam insertVPPAppTeams transaction") } @@ -656,7 +656,7 @@ func (ds *Datastore) InsertVPPAppWithTeam(ctx context.Context, app *fleet.VPPApp app.VPPAppTeam.AppTeamID = vppAppTeamID if app.ValidatedLabels != nil { - if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, vppAppTeamID, *app.ValidatedLabels, softwareTypeVPP); err != nil { + if err := setOrUpdateSoftwareInstallerLabelsDB(ctx, tx, ds.dialect, vppAppTeamID, *app.ValidatedLabels, softwareTypeVPP); err != nil { return ctxerr.Wrap(ctx, err, "InsertVPPAppWithTeam setOrUpdateSoftwareInstallerLabelsDB transaction") } } @@ -685,7 +685,7 @@ func (ds *Datastore) InsertVPPAppWithTeam(ctx context.Context, app *fleet.VPPApp } if app.DisplayName != nil { - if err := updateSoftwareTitleDisplayName(ctx, tx, teamID, titleID, *app.DisplayName); err != nil { + if err := updateSoftwareTitleDisplayName(ctx, tx, ds.dialect, teamID, titleID, *app.DisplayName); err != nil { return ctxerr.Wrap(ctx, err, "setting software title display name for vpp app") } } @@ -752,23 +752,22 @@ WHERE func (ds *Datastore) InsertVPPApps(ctx context.Context, apps []*fleet.VPPApp) error { return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - return insertVPPApps(ctx, tx, apps) + return insertVPPApps(ctx, tx, ds.dialect, apps) }) } -func insertVPPApps(ctx context.Context, tx sqlx.ExtContext, apps []*fleet.VPPApp) error { +func insertVPPApps(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, apps []*fleet.VPPApp) error { stmt := ` INSERT INTO vpp_apps (adam_id, bundle_identifier, icon_url, name, latest_version, title_id, platform) VALUES %s -ON DUPLICATE KEY UPDATE +` + dialect.OnDuplicateKey("adam_id,platform", ` updated_at = CURRENT_TIMESTAMP, latest_version = VALUES(latest_version), icon_url = VALUES(icon_url), name = VALUES(name), - title_id = VALUES(title_id) - ` + title_id = VALUES(title_id)`) var args []any var insertVals strings.Builder @@ -784,16 +783,15 @@ ON DUPLICATE KEY UPDATE return ctxerr.Wrap(ctx, err, "insert VPP apps") } -func insertVPPAppTeams(ctx context.Context, tx sqlx.ExtContext, appID fleet.VPPAppTeam, teamID *uint, vppTokenID *uint) (uint, error) { +func insertVPPAppTeams(ctx context.Context, tx sqlx.ExtContext, dialect DialectHelper, appID fleet.VPPAppTeam, teamID *uint, vppTokenID *uint) (uint, error) { stmt := ` INSERT INTO vpp_apps_teams (adam_id, global_or_team_id, team_id, platform, self_service, vpp_token_id, install_during_setup) VALUES (?, ?, ?, ?, ?, ?, COALESCE(?, false)) -ON DUPLICATE KEY UPDATE +` + dialect.OnDuplicateKey("global_or_team_id, adam_id, platform", ` self_service = VALUES(self_service), - install_during_setup = COALESCE(?, install_during_setup) -` + install_during_setup = COALESCE(?, install_during_setup)`) var globalOrTmID uint if teamID != nil { @@ -818,8 +816,9 @@ ON DUPLICATE KEY UPDATE var id int64 if insertOnDuplicateDidInsertOrUpdate(res) { - id, _ = res.LastInsertId() - } else { + id, _ = res.LastInsertId() // PG: returns 0, fallback below + } + if id == 0 { stmt := `SELECT id FROM vpp_apps_teams WHERE adam_id = ? AND platform = ? AND global_or_team_id = ?` if err := sqlx.GetContext(ctx, tx, &id, stmt, appID.AdamID, appID.Platform, globalOrTmID); err != nil { return 0, ctxerr.Wrap(ctx, err, "vpp app teams id") @@ -898,7 +897,7 @@ func (ds *Datastore) getOrInsertSoftwareTitleForVPPApp(ctx context.Context, tx s selectStmt = ` SELECT id FROM software_titles - WHERE bundle_identifier = ? AND additional_identifier = 0` + WHERE bundle_identifier = ? AND (additional_identifier IS NULL OR additional_identifier = '0')` selectArgs = []any{app.BundleIdentifier} } } @@ -932,7 +931,7 @@ func (ds *Datastore) DeleteVPPAppFromTeam(ctx context.Context, teamID *uint, app tx := ds.writer(ctx) // make sure we're looking at a consistent vision of the world when deleting res, err := tx.ExecContext(ctx, stmt, globalOrTeamID, appID.AdamID, appID.Platform) if err != nil { - if isMySQLForeignKey(err) { + if ds.dialect.IsForeignKey(err) { // Check if the app is referenced by a policy automation. var count int if err := sqlx.GetContext(ctx, tx, &count, `SELECT COUNT(*) FROM policies p JOIN vpp_apps_teams vat @@ -1118,7 +1117,7 @@ VALUES } err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - res, err := tx.ExecContext(ctx, insertUAStmt, + activityID, err := insertAndGetIDTx(ctx, tx, ds.dialect, insertUAStmt, hostID, opts.Priority(), userID, @@ -1132,8 +1131,6 @@ VALUES if err != nil { return ctxerr.Wrap(ctx, err, "insert vpp install request") } - - activityID, _ := res.LastInsertId() _, err = tx.ExecContext(ctx, insertVAUAStmt, activityID, appID.AdamID, @@ -1387,9 +1384,7 @@ func (ds *Datastore) InsertVPPToken(ctx context.Context, tok *fleet.VPPTokenData return nil, ctxerr.Wrap(ctx, err, "encrypt token with datastore.serverPrivateKey") } - res, err := ds.writer(ctx).ExecContext( - ctx, - insertStmt, + id, err := ds.insertAndGetID(ctx, ds.writer(ctx), insertStmt, vppTokenDB.OrgName, vppTokenDB.Location, vppTokenDB.RenewDate, @@ -1399,8 +1394,6 @@ func (ds *Datastore) InsertVPPToken(ctx context.Context, tok *fleet.VPPTokenData return nil, ctxerr.Wrap(ctx, err, "inserting vpp token") } - id, _ := res.LastInsertId() - vppTokenDB.ID = uint(id) //nolint:gosec // dismiss G115 return vppTokenDB, nil @@ -1668,7 +1661,7 @@ func (ds *Datastore) UpdateVPPTokenTeams(ctx context.Context, id uint, teams []u if err != nil { var mysqlErr *mysql.MySQLError // https://dev.mysql.com/doc/mysql-errors/8.4/en/server-error-reference.html#error_er_dup_entry - if errors.As(err, &mysqlErr) && IsDuplicate(err) { + if errors.As(err, &mysqlErr) && ds.dialect.IsDuplicate(err) { var dupeTeamID uint var dupeTeamName string _, _ = fmt.Sscanf(mysqlErr.Message, "Duplicate entry '%d' for", &dupeTeamID) @@ -2156,20 +2149,22 @@ func (ds *Datastore) MarkAllPendingAppleVPPAndInHouseInstallsAsFailed(ctx contex // but those in host_vpp_software_installs could be Android as well. clearVPPUpcomingActivitiesStmt := ` -DELETE ua FROM - upcoming_activities ua -JOIN - host_vpp_software_installs hvsi ON hvsi.command_uuid = ua.execution_id -WHERE ua.activity_type = ? AND hvsi.verification_failed_at IS NULL -AND hvsi.verification_at IS NULL AND hvsi.platform != 'android' +DELETE FROM upcoming_activities +WHERE upcoming_activities.activity_type = ? AND EXISTS ( + SELECT 1 FROM host_vpp_software_installs hvsi + WHERE hvsi.command_uuid = upcoming_activities.execution_id + AND hvsi.verification_failed_at IS NULL + AND hvsi.verification_at IS NULL AND hvsi.platform != 'android' +) ` clearInHouseUpcomingActivitiesStmt := ` -DELETE ua FROM - upcoming_activities ua -JOIN - host_in_house_software_installs hihs ON hihs.command_uuid = ua.execution_id -WHERE ua.activity_type = ? AND hihs.verification_failed_at IS NULL AND hihs.verification_at IS NULL +DELETE FROM upcoming_activities +WHERE upcoming_activities.activity_type = ? AND EXISTS ( + SELECT 1 FROM host_in_house_software_installs hihs + WHERE hihs.command_uuid = upcoming_activities.execution_id + AND hihs.verification_failed_at IS NULL AND hihs.verification_at IS NULL +) ` installVPPFailStmt := ` @@ -2519,13 +2514,13 @@ func (ds *Datastore) hasAppStoreAppChanged(ctx context.Context, teamID *uint, in } func (ds *Datastore) IsAutoUpdateVPPInstall(ctx context.Context, commandUUID string) (bool, error) { - stmt := ` + stmt := fmt.Sprintf(` SELECT COUNT(*) > 0 FROM upcoming_activities WHERE execution_id = ? AND activity_type = 'vpp_app_install' - AND JSON_EXTRACT(payload, '$.from_auto_update') = 1 -` + AND %s = 1 +`, ds.dialect.JSONExtract("payload", "$.from_auto_update")) var isAutoUpdate bool if err := sqlx.GetContext(ctx, ds.reader(ctx), &isAutoUpdate, stmt, commandUUID); err != nil { return false, ctxerr.Wrap(ctx, err, "checking if vpp install is from auto update") diff --git a/server/datastore/mysql/windows_updates.go b/server/datastore/mysql/windows_updates.go index d364f48a70d..0f17bf5edd5 100644 --- a/server/datastore/mysql/windows_updates.go +++ b/server/datastore/mysql/windows_updates.go @@ -60,7 +60,7 @@ func (ds *Datastore) InsertWindowsUpdates(ctx context.Context, hostID uint, upda if len(args) > 0 { smt := fmt.Sprintf( - `INSERT IGNORE INTO windows_updates (host_id, date_epoch, kb_id) VALUES %s`, + ds.dialect.InsertIgnoreInto()+` windows_updates (host_id, date_epoch, kb_id) VALUES %s`+ds.dialect.OnConflictDoNothing("host_id,date_epoch,kb_id"), strings.Join(placeholders, ","), ) diff --git a/server/datastore/mysql/wstep.go b/server/datastore/mysql/wstep.go index ebcd43468eb..bc1393c45a0 100644 --- a/server/datastore/mysql/wstep.go +++ b/server/datastore/mysql/wstep.go @@ -45,15 +45,11 @@ VALUES // WSTEPNewSerial allocates and returns a new (increasing) serial number. func (ds *Datastore) WSTEPNewSerial(ctx context.Context) (*big.Int, error) { - result, err := ds.writer(ctx).ExecContext(ctx, `INSERT INTO wstep_serials () VALUES ();`) + lid, err := ds.insertAndGetID(ctx, ds.writer(ctx), `INSERT INTO wstep_serials () VALUES ();`) if err != nil { return nil, err } - lid, err := result.LastInsertId() // TODO: ok if sequential and not random? - if err != nil { - return nil, err - } - // TODO: check maxSerialNumber? + // TODO: check maxSerialNumber? ok if sequential and not random? return big.NewInt(lid), nil } diff --git a/server/platform/endpointer/endpoint_utils.go b/server/platform/endpointer/endpoint_utils.go index 26d462025c6..b013edd3efb 100644 --- a/server/platform/endpointer/endpoint_utils.go +++ b/server/platform/endpointer/endpoint_utils.go @@ -546,8 +546,6 @@ func MakeDecoder( return nil, inner } - // This is the DecodeRequest implementation returning http.MaxBytesError - // (e.g. there's a size limit when uploading installers.) if _, isMaxBytesError := errors.AsType[*http.MaxBytesError](err); isMaxBytesError { return nil, platform_http.PayloadTooLargeError{ ContentLength: r.Header.Get("Content-Length"), diff --git a/server/platform/mysql/common.go b/server/platform/mysql/common.go index bf7349b9b3f..19fd0f810ee 100644 --- a/server/platform/mysql/common.go +++ b/server/platform/mysql/common.go @@ -223,10 +223,16 @@ func WithTxx(ctx context.Context, db *sqlx.DB, fn TxFn, logger *slog.Logger) err // WithReadOnlyTxx executes fn within an isolated, read-only transaction func WithReadOnlyTxx(ctx context.Context, reader *sqlx.DB, fn ReadTxFn, logger *slog.Logger) error { - tx, err := reader.BeginTxx(ctx, &sql.TxOptions{ + txOpts := &sql.TxOptions{ ReadOnly: true, Isolation: sql.LevelRepeatableRead, - }) + } + // pgx does not support non-default isolation levels via database/sql's + // TxOptions, so fall back to LevelDefault for PostgreSQL connections. + if reader.DriverName() == "pgx" || reader.DriverName() == "pgx-rebind" { + txOpts.Isolation = sql.LevelDefault + } + tx, err := reader.BeginTxx(ctx, txOpts) if err != nil { return ctxerr.Wrap(ctx, err, "create read-only transaction") } diff --git a/server/platform/mysql/list_options.go b/server/platform/mysql/list_options.go index d0865496187..dc6bf9677a4 100644 --- a/server/platform/mysql/list_options.go +++ b/server/platform/mysql/list_options.go @@ -4,6 +4,7 @@ import ( "fmt" "regexp" "sort" + "strconv" "strings" ) @@ -124,7 +125,12 @@ func AppendListOptionsWithParamsSecure(sql string, params []any, opts ListOption // Cursor value is always passed as string. MySQL automatically converts // string to integer when comparing against integer columns. // See: https://dev.mysql.com/doc/refman/8.0/en/type-conversion.html - params = append(params, cursor) + // PG does NOT auto-convert, so pass numeric cursors as int64. + var cursorParam any = cursor + if v, err := strconv.ParseInt(cursor, 10, 64); err == nil { + cursorParam = v + } + params = append(params, cursorParam) direction := ">" // ASC if opts.IsDescending() { direction = "<" // DESC @@ -188,7 +194,12 @@ func AppendListOptionsWithParams(sql string, params []any, opts ListOptions) (st // Cursor value is always passed as string. MySQL automatically converts // string to integer when comparing against integer columns. // See: https://dev.mysql.com/doc/refman/8.0/en/type-conversion.html - params = append(params, cursor) + // PG does NOT auto-convert, so pass numeric cursors as int64. + var cursorParam any = cursor + if v, err := strconv.ParseInt(cursor, 10, 64); err == nil { + cursorParam = v + } + params = append(params, cursorParam) direction := ">" // ASC if opts.IsDescending() { direction = "<" // DESC diff --git a/server/platform/mysql/testing_utils/testing_utils.go b/server/platform/mysql/testing_utils/testing_utils.go index a48c6103ecb..dac63a448c7 100644 --- a/server/platform/mysql/testing_utils/testing_utils.go +++ b/server/platform/mysql/testing_utils/testing_utils.go @@ -57,12 +57,26 @@ func TruncateTables(t testing.TB, db *sqlx.DB, logger *slog.Logger, nonEmptyTabl ctx := context.Background() + isPG := strings.Contains(db.DriverName(), "pgx") + require.NoError(t, common_mysql.WithTxx(ctx, db, func(tx sqlx.ExtContext) error { var skipSeeded bool if len(tables) == 0 { skipSeeded = true - sql := ` + var sql string + if isPG { + sql = ` + SELECT + table_name + FROM + information_schema.tables + WHERE + table_schema = current_schema() AND + table_type = 'BASE TABLE' + ` + } else { + sql = ` SELECT table_name FROM @@ -71,13 +85,20 @@ func TruncateTables(t testing.TB, db *sqlx.DB, logger *slog.Logger, nonEmptyTabl table_schema = database() AND table_type = 'BASE TABLE' ` + } if err := sqlx.SelectContext(ctx, tx, &tables, sql); err != nil { return err } } - if _, err := tx.ExecContext(ctx, `SET FOREIGN_KEY_CHECKS=0`); err != nil { - return err + if isPG { + if _, err := tx.ExecContext(ctx, `SET session_replication_role = 'replica'`); err != nil { + return err + } + } else { + if _, err := tx.ExecContext(ctx, `SET FOREIGN_KEY_CHECKS=0`); err != nil { + return err + } } for _, tbl := range tables { if nonEmptyTables[tbl] { @@ -86,12 +107,22 @@ func TruncateTables(t testing.TB, db *sqlx.DB, logger *slog.Logger, nonEmptyTabl } return fmt.Errorf("cannot truncate table %s, it contains seed data from schema.sql", tbl) } - if _, err := tx.ExecContext(ctx, "TRUNCATE TABLE "+tbl); err != nil { + truncateSQL := "TRUNCATE TABLE " + tbl + if isPG { + truncateSQL += " CASCADE" + } + if _, err := tx.ExecContext(ctx, truncateSQL); err != nil { return err } } - if _, err := tx.ExecContext(ctx, `SET FOREIGN_KEY_CHECKS=1`); err != nil { - return err + if isPG { + if _, err := tx.ExecContext(ctx, `SET session_replication_role = 'origin'`); err != nil { + return err + } + } else { + if _, err := tx.ExecContext(ctx, `SET FOREIGN_KEY_CHECKS=1`); err != nil { + return err + } } return nil }, logger)) From 03cd9551d28fa5087f25ec13f224946ba3be2d68 Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Wed, 1 Apr 2026 09:31:33 -0400 Subject: [PATCH 4/6] test(datastore): fix test helpers and assertions for PostgreSQL Update test files to pass ds.dialect to helper functions that now require it for dialect-aware SQL generation: - mdm_test.go: batchSetProfileLabelAssociationsDB calls - microsoft_mdm_test.go: batchSetProfileVariableAssociationsDB calls - operating_systems_test.go: upsertHostOperatingSystemDB calls - packs_test.go: saveHostPackStatsDB calls - policies_test.go: deleteAllPolicyMemberships, cleanupPolicyMembership* calls - scripts_test.go: insertScriptContents return type int64 (was sql.Result) - software_test.go: setOrUpdateSoftwareInstallerLabelsDB calls --- server/datastore/mysql/mdm_test.go | 25 +++++----- server/datastore/mysql/microsoft_mdm_test.go | 4 +- .../datastore/mysql/operating_systems_test.go | 14 +++--- server/datastore/mysql/packs_test.go | 4 +- server/datastore/mysql/policies_test.go | 8 ++-- server/datastore/mysql/scripts_test.go | 12 ++--- server/datastore/mysql/software_test.go | 46 +++++++++---------- 7 files changed, 57 insertions(+), 56 deletions(-) diff --git a/server/datastore/mysql/mdm_test.go b/server/datastore/mysql/mdm_test.go index 65292f42815..22c9b0813a2 100644 --- a/server/datastore/mysql/mdm_test.go +++ b/server/datastore/mysql/mdm_test.go @@ -6943,14 +6943,14 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { wantOtherWin := []fleet.ConfigurationProfileLabel{ {ProfileUUID: otherWinProfile.ProfileUUID, LabelName: label.Name, LabelID: label.ID}, } - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, ds.writer(ctx), wantOtherWin, []string{windowsProfile.ProfileUUID}, "windows") + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, ds.writer(ctx), ds.dialect, wantOtherWin, []string{windowsProfile.ProfileUUID}, "windows") require.NoError(t, err) assert.True(t, updatedDB) // make it an "exclude" label on the other macos profile wantOtherMac := []fleet.ConfigurationProfileLabel{ {ProfileUUID: otherMacProfile.ProfileUUID, LabelName: label.Name, LabelID: label.ID, Exclude: true}, } - updatedDB, err = batchSetProfileLabelAssociationsDB(ctx, ds.writer(ctx), wantOtherMac, []string{macOSProfile.ProfileUUID}, "darwin") + updatedDB, err = batchSetProfileLabelAssociationsDB(ctx, ds.writer(ctx), ds.dialect, wantOtherMac, []string{macOSProfile.ProfileUUID}, "darwin") require.NoError(t, err) assert.True(t, updatedDB) @@ -6985,7 +6985,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { t.Run("empty input "+platform, func(t *testing.T) { want := []fleet.ConfigurationProfileLabel{} err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, want, nil, platform) + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, want, nil, platform) require.NoError(t, err) assert.False(t, updatedDB) return err @@ -7002,7 +7002,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { {ProfileUUID: uuid, LabelName: label.Name, LabelID: label.ID}, } err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, profileLabels, nil, platform) + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, profileLabels, nil, platform) require.NoError(t, err) assert.True(t, updatedDB) return err @@ -7018,7 +7018,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { {ProfileUUID: uuid, LabelName: label.Name, LabelID: label.ID, Exclude: true}, } err = ds.withTx(ctx, func(tx sqlx.ExtContext) error { - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, profileLabels, nil, platform) + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, profileLabels, nil, platform) require.NoError(t, err) assert.True(t, updatedDB) return err @@ -7036,7 +7036,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { } err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { - _, err := batchSetProfileLabelAssociationsDB(ctx, tx, invalidProfileLabels, nil, platform) + _, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, invalidProfileLabels, nil, platform) return err }) require.Error(t, err) @@ -7048,7 +7048,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { {ProfileUUID: uuid, LabelName: label.Name, LabelID: 12345}, } err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { - _, err := batchSetProfileLabelAssociationsDB(ctx, tx, invalidProfileLabels, nil, platform) + _, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, invalidProfileLabels, nil, platform) return err }) require.Error(t, err) @@ -7058,7 +7058,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { {ProfileUUID: uuid, LabelName: "xyz", LabelID: 1235}, } err = ds.withTx(ctx, func(tx sqlx.ExtContext) error { - _, err := batchSetProfileLabelAssociationsDB(ctx, tx, invalidProfileLabels, nil, platform) + _, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, invalidProfileLabels, nil, platform) return err }) require.Error(t, err) @@ -7080,7 +7080,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { {ProfileUUID: uuid, LabelName: newLabel.Name, LabelID: newLabel.ID, Exclude: true}, } err = ds.withTx(ctx, func(tx sqlx.ExtContext) error { - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, profileLabels, nil, platform) + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, profileLabels, nil, platform) require.NoError(t, err) assert.True(t, updatedDB) return err @@ -7094,7 +7094,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { {ProfileUUID: uuid, LabelName: label.Name, LabelID: label.ID}, } err = ds.withTx(ctx, func(tx sqlx.ExtContext) error { - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, profileLabels, nil, platform) + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, profileLabels, nil, platform) require.NoError(t, err) assert.True(t, updatedDB) return err @@ -7104,7 +7104,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { // batch apply again this time without any label err = ds.withTx(ctx, func(tx sqlx.ExtContext) error { - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, nil, []string{uuid}, platform) + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, nil, []string{uuid}, platform) require.NoError(t, err) assert.True(t, updatedDB) return err @@ -7118,7 +7118,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { // batch apply again with no change returns false err = ds.withTx(ctx, func(tx sqlx.ExtContext) error { - updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, nil, []string{uuid}, platform) + updatedDB, err := batchSetProfileLabelAssociationsDB(ctx, tx, ds.dialect, nil, []string{uuid}, platform) require.NoError(t, err) assert.False(t, updatedDB) return err @@ -7133,6 +7133,7 @@ func testBatchSetProfileLabelAssociations(t *testing.T, ds *Datastore) { _, err := batchSetProfileLabelAssociationsDB( ctx, tx, + ds.dialect, []fleet.ConfigurationProfileLabel{{}}, nil, "unsupported", diff --git a/server/datastore/mysql/microsoft_mdm_test.go b/server/datastore/mysql/microsoft_mdm_test.go index 1a0107c14ff..17c248962dd 100644 --- a/server/datastore/mysql/microsoft_mdm_test.go +++ b/server/datastore/mysql/microsoft_mdm_test.go @@ -3567,7 +3567,7 @@ func testSetMDMWindowsProfilesWithVariables(t *testing.T, ds *Datastore) { } // both profiles have no variable - _, err := batchSetProfileVariableAssociationsDB(ctx, ds.writer(ctx), []fleet.MDMProfileUUIDFleetVariables{ + _, err := batchSetProfileVariableAssociationsDB(ctx, ds.writer(ctx), ds.dialect, []fleet.MDMProfileUUIDFleetVariables{ {ProfileUUID: globalProfiles[0], FleetVariables: nil}, {ProfileUUID: globalProfiles[1], FleetVariables: nil}, }, "windows") @@ -3577,7 +3577,7 @@ func testSetMDMWindowsProfilesWithVariables(t *testing.T, ds *Datastore) { checkProfileVariables(globalProfiles[1], 0, nil) // add some variables - _, err = batchSetProfileVariableAssociationsDB(ctx, ds.writer(ctx), []fleet.MDMProfileUUIDFleetVariables{ + _, err = batchSetProfileVariableAssociationsDB(ctx, ds.writer(ctx), ds.dialect, []fleet.MDMProfileUUIDFleetVariables{ {ProfileUUID: globalProfiles[0], FleetVariables: []fleet.FleetVarName{fleet.FleetVarHostEndUserIDPUsername, fleet.FleetVarName(string(fleet.FleetVarDigiCertDataPrefix) + "ZZZ")}}, {ProfileUUID: globalProfiles[1], FleetVariables: []fleet.FleetVarName{fleet.FleetVarHostEndUserIDPGroups}}, }, "windows") diff --git a/server/datastore/mysql/operating_systems_test.go b/server/datastore/mysql/operating_systems_test.go index d67b7261784..3645fd2f9dc 100644 --- a/server/datastore/mysql/operating_systems_test.go +++ b/server/datastore/mysql/operating_systems_test.go @@ -261,21 +261,21 @@ func TestMaybeUpdateHostOperatingSystem(t *testing.T) { require.ErrorIs(t, err, sql.ErrNoRows) // insert test host and os id - err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID, osList[0].ID) + err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), ds.dialect, testHostID, osList[0].ID) require.NoError(t, err) osID, err := getIDHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID) require.NoError(t, err) require.Equal(t, osList[0].ID, osID) // update test host with new os id - err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID, osList[1].ID) + err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), ds.dialect, testHostID, osList[1].ID) require.NoError(t, err) osID, err = getIDHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID) require.NoError(t, err) require.Equal(t, osList[1].ID, osID) // no change - err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID, osList[1].ID) + err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), ds.dialect, testHostID, osList[1].ID) require.NoError(t, err) osID, err = getIDHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID) require.NoError(t, err) @@ -300,7 +300,7 @@ func TestGetHostOperatingSystem(t *testing.T) { require.ErrorIs(t, err, sql.ErrNoRows) // insert test host and os id - err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID, osList[0].ID) + err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), ds.dialect, testHostID, osList[0].ID) require.NoError(t, err) os, err := getHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID) require.NoError(t, err) @@ -311,7 +311,7 @@ func TestGetHostOperatingSystem(t *testing.T) { require.Equal(t, osList[0], *os) // update test host with new os id - err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID, osList[1].ID) + err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), ds.dialect, testHostID, osList[1].ID) require.NoError(t, err) os, err = getHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID) require.NoError(t, err) @@ -322,7 +322,7 @@ func TestGetHostOperatingSystem(t *testing.T) { require.Equal(t, osList[1], *os) // no change - err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID, osList[1].ID) + err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), ds.dialect, testHostID, osList[1].ID) require.NoError(t, err) os, err = getHostOperatingSystemDB(ctx, ds.writer(ctx), testHostID) require.NoError(t, err) @@ -360,7 +360,7 @@ func TestCleanupHostOperatingSystems(t *testing.T) { // insert host operating system record so initially each os is seeded with two hosts hostOS := testOSs[i%len(testOSs)] - err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), h.ID, hostOS.ID) + err = upsertHostOperatingSystemDB(ctx, ds.writer(ctx), ds.dialect, h.ID, hostOS.ID) require.NoError(t, err) osByHostID[h.ID] = hostOS } diff --git a/server/datastore/mysql/packs_test.go b/server/datastore/mysql/packs_test.go index 2350e730e7d..d02d1370c08 100644 --- a/server/datastore/mysql/packs_test.go +++ b/server/datastore/mysql/packs_test.go @@ -498,7 +498,7 @@ func testPacksApplyStatsNotLocking(t *testing.T, ds *Datastore) { require.NoError(t, err) amount := rand.Intn(5000) - require.NoError(t, saveHostPackStatsDB(context.Background(), ds.writer(context.Background()), host.TeamID, host.ID, randomPackStatsForHost(pack.ID, pack.Name, *pack.Type, schedQueries, amount))) + require.NoError(t, saveHostPackStatsDB(context.Background(), ds.writer(context.Background()), ds.dialect, host.TeamID, host.ID, randomPackStatsForHost(pack.ID, pack.Name, *pack.Type, schedQueries, amount))) } } }() @@ -550,7 +550,7 @@ func testPacksApplyStatsNotLockingTryTwo(t *testing.T, ds *Datastore) { require.NoError(t, err) amount := rand.Intn(5000) - require.NoError(t, saveHostPackStatsDB(context.Background(), ds.writer(context.Background()), host.TeamID, host.ID, randomPackStatsForHost(pack.ID, pack.Name, *pack.Type, schedQueries, amount))) + require.NoError(t, saveHostPackStatsDB(context.Background(), ds.writer(context.Background()), ds.dialect, host.TeamID, host.ID, randomPackStatsForHost(pack.ID, pack.Name, *pack.Type, schedQueries, amount))) } } }() diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go index 0c7bd38260f..2ae0fdbc9ae 100644 --- a/server/datastore/mysql/policies_test.go +++ b/server/datastore/mysql/policies_test.go @@ -3331,7 +3331,7 @@ func testDeleteAllPolicyMemberships(t *testing.T, ds *Datastore) { require.NoError(t, ds.writer(ctx).Get(&count, "select COUNT(*) from host_issues WHERE total_issues_count > 0")) assert.Equal(t, 1, count) - err = deleteAllPolicyMemberships(ctx, ds.writer(ctx), host.ID) + err = deleteAllPolicyMemberships(ctx, ds.writer(ctx), ds.dialect, host.ID) require.NoError(t, err) err = ds.writer(ctx).Get(&count, "select COUNT(*) from policy_membership") @@ -7165,7 +7165,7 @@ func testBatchedPolicyMembershipCleanup(t *testing.T, ds *Datastore) { // Run the full cleanup function directly (simulates what ApplyPolicySpecs triggers when a // query changes — shouldRemoveAllPolicyMemberships == true). - err = cleanupPolicyMembershipForPolicy(ctx, ds.reader(ctx), ds.writer(ctx), pol.ID) + err = cleanupPolicyMembershipForPolicy(ctx, ds.reader(ctx), ds.writer(ctx), ds.dialect, pol.ID) require.NoError(t, err) // All policy_membership rows must be gone. @@ -7237,7 +7237,7 @@ func testBatchedPolicyMembershipCleanupOnPolicyUpdate(t *testing.T, ds *Datastor require.Equal(t, 6, count) // Run the platform-aware cleanup (simulates CleanupPolicyMembership cron). - err = cleanupPolicyMembershipOnPolicyUpdate(ctx, ds.reader(ctx), ds.writer(ctx), pol.ID, pol.Platform) + err = cleanupPolicyMembershipOnPolicyUpdate(ctx, ds.reader(ctx), ds.writer(ctx), pol.ID, pol.Platform, ds.dialect) require.NoError(t, err) // Only the windows host should remain. @@ -7302,7 +7302,7 @@ func testBatchedPolicyMembershipCleanupOnPolicyUpdate(t *testing.T, ds *Datastor // Run cleanupPolicyMembershipOnPolicyUpdate with no platform restriction so // only the label-based branch fires. - err = cleanupPolicyMembershipOnPolicyUpdate(ctx, ds.reader(ctx), ds.writer(ctx), lblPol.ID, "" /* no platform filter */) + err = cleanupPolicyMembershipOnPolicyUpdate(ctx, ds.reader(ctx), ds.writer(ctx), lblPol.ID, "" /* no platform filter */, ds.dialect) require.NoError(t, err) // Only the host that belongs to the include label should remain. diff --git a/server/datastore/mysql/scripts_test.go b/server/datastore/mysql/scripts_test.go index c464532111b..4d4c54fd358 100644 --- a/server/datastore/mysql/scripts_test.go +++ b/server/datastore/mysql/scripts_test.go @@ -1378,16 +1378,16 @@ type scriptContents struct { func testInsertScriptContents(t *testing.T, ds *Datastore) { ctx := context.Background() contents := `echo foobar;` - res, err := insertScriptContents(ctx, ds.writer(ctx), contents) + res, err := insertScriptContents(ctx, ds.writer(ctx), ds.dialect, contents) require.NoError(t, err) - id, _ := res.LastInsertId() + id := res require.Equal(t, int64(1), id) expectedCS := md5ChecksumScriptContent(contents) // insert same contents again, verify that the checksum and ID stayed the same - res, err = insertScriptContents(ctx, ds.writer(ctx), contents) + res, err = insertScriptContents(ctx, ds.writer(ctx), ds.dialect, contents) require.NoError(t, err) - id, _ = res.LastInsertId() + id = res require.Equal(t, int64(1), id) stmt := `SELECT id, HEX(md5_checksum) as md5_checksum FROM script_contents WHERE id = ?` @@ -1523,9 +1523,9 @@ func testCleanupUnusedScriptContents(t *testing.T, ds *Datastore) { func testGetAnyScriptContents(t *testing.T, ds *Datastore) { ctx := context.Background() contents := `echo foobar;` - res, err := insertScriptContents(ctx, ds.writer(ctx), contents) + res, err := insertScriptContents(ctx, ds.writer(ctx), ds.dialect, contents) require.NoError(t, err) - id, _ := res.LastInsertId() + id := res result, err := ds.GetAnyScriptContents(ctx, uint(id)) //nolint:gosec // dismiss G115 require.NoError(t, err) diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index 6d73a2392e5..3612375126f 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -7814,12 +7814,12 @@ func testListHostSoftwareWithLabelScoping(t *testing.T, ds *Datastore) { time.Sleep(time.Second) // assign the label to the software installers - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID1, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID1, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeInstaller) require.NoError(t, err) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), selfServiceInstallerID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, selfServiceInstallerID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeInstaller) @@ -7872,7 +7872,7 @@ func testListHostSoftwareWithLabelScoping(t *testing.T, ds *Datastore) { require.Len(t, software, 0) // Update the label to be "include any" - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID1, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID1, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeInstaller) @@ -7926,7 +7926,7 @@ func testListHostSoftwareWithLabelScoping(t *testing.T, ds *Datastore) { label3, err := ds.NewLabel(ctx, &fleet.Label{Name: "label3" + t.Name(), LabelMembershipType: fleet.LabelMembershipTypeManual}) require.NoError(t, err) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID2, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID2, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{ label2.Name: {LabelName: label2.Name, LabelID: label2.ID}, @@ -7989,7 +7989,7 @@ func testListHostSoftwareWithLabelScoping(t *testing.T, ds *Datastore) { label4, err := ds.NewLabel(ctx, &fleet.Label{Name: "label4" + t.Name(), LabelMembershipType: fleet.LabelMembershipTypeDynamic}) require.NoError(t, err) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID3, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID3, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label4.Name: {LabelName: label4.Name, LabelID: label4.ID}}, }, softwareTypeInstaller) @@ -8033,7 +8033,7 @@ func testListHostSoftwareWithLabelScoping(t *testing.T, ds *Datastore) { require.True(t, scoped) // Now include hosts with label4. No host has this label, so we shouldn't see installerID3 anymore. - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID3, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID3, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{label4.Name: {LabelName: label4.Name, LabelID: label4.ID}}, }, softwareTypeInstaller) @@ -8083,7 +8083,7 @@ func testListHostSoftwareWithLabelScoping(t *testing.T, ds *Datastore) { label5, err := ds.NewLabel(ctx, &fleet.Label{Name: "label5" + t.Name(), LabelMembershipType: fleet.LabelMembershipTypeManual}) require.NoError(t, err) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID4, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID4, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label5.Name: {LabelName: label5.Name, LabelID: label5.ID}}, }, softwareTypeInstaller) @@ -8103,7 +8103,7 @@ func testListHostSoftwareWithLabelScoping(t *testing.T, ds *Datastore) { // Scope installer1 to include_all: [label1, label4]. // hostIncludeAll has neither label yet, so installer1 should be out of scope. - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID1, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID1, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAll, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}, label4.Name: {LabelName: label4.Name, LabelID: label4.ID}}, }, softwareTypeInstaller) @@ -9015,7 +9015,7 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { time.Sleep(time.Second) // assign the label to the software installer - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID1, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID1, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeInstaller) @@ -9095,7 +9095,7 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { require.True(t, scoped) // Assign the label to the VPP app. Now we should have an empty list - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vppAppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vppAppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeVPP) @@ -9177,13 +9177,13 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { opts.OnlyAvailableForInstall = false // Make the label include any. We should have both of them back. - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), installerID1, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, installerID1, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeInstaller) require.NoError(t, err) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vppAppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vppAppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeVPP) @@ -9195,7 +9195,7 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { // Give the VPP app a different label. Only the installer should show up now, since the host // only has label1. - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vppAppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vppAppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{label2.Name: {LabelName: label2.Name, LabelID: label2.ID}}, }, softwareTypeVPP) @@ -9209,7 +9209,7 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { require.NoError(t, err) require.False(t, scoped) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vppAppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vppAppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{label2.Name: {LabelName: label2.Name, LabelID: label2.ID}, label1.Name: {LabelName: label1.Name, LabelID: label1.ID}}, }, softwareTypeVPP) @@ -9228,7 +9228,7 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { label3, err := ds.NewLabel(ctx, &fleet.Label{Name: "label3" + t.Name()}) require.NoError(t, err) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vppAppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vppAppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label3.Name: {LabelName: label3.Name, LabelID: label3.ID}}, }, softwareTypeVPP) @@ -9262,7 +9262,7 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { label4, err := ds.NewLabel(ctx, &fleet.Label{Name: "label4" + t.Name(), LabelMembershipType: fleet.LabelMembershipTypeManual}) require.NoError(t, err) - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vppAppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vppAppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{label4.Name: {LabelName: label4.Name, LabelID: label4.ID}}, }, softwareTypeVPP) @@ -9283,7 +9283,7 @@ func testListHostSoftwareWithLabelScopingVPP(t *testing.T, ds *Datastore) { // Scope the VPP app to include_all: [label5, label6]. // host currently has label1 but not label5 or label6. - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vppAppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vppAppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAll, ByName: map[string]fleet.LabelIdent{ label5.Name: {LabelName: label5.Name, LabelID: label5.ID}, @@ -9590,13 +9590,13 @@ func testListHostSoftwareSelfServiceWithLabelScopingHostInstalled(t *testing.T, err = ds.UpdateHost(ctx, host) require.NoError(t, err) // label software - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), selfServiceInstallerID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, selfServiceInstallerID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{excludeLabel.Name: {LabelName: excludeLabel.Name, LabelID: excludeLabel.ID}}, }, softwareTypeInstaller) require.NoError(t, err) // label vpp app - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), vPPApp.VPPAppTeam.AppTeamID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, vPPApp.VPPAppTeam.AppTeamID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{excludeLabel.Name: {LabelName: excludeLabel.Name, LabelID: excludeLabel.ID}}, }, softwareTypeVPP) @@ -9859,7 +9859,7 @@ func testLabelScopingTimestampLogic(t *testing.T, ds *Datastore) { } // Dynamic label exclude any - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), selfServiceInstallerID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, selfServiceInstallerID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{ label1.Name: {LabelName: label1.Name, LabelID: label1.ID}, @@ -9878,7 +9878,7 @@ func testLabelScopingTimestampLogic(t *testing.T, ds *Datastore) { require.Len(t, software, 0) // manual label exclude any - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), selfServiceInstallerID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, selfServiceInstallerID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeExcludeAny, ByName: map[string]fleet.LabelIdent{ label2.Name: {LabelName: label2.Name, LabelID: label2.ID}, @@ -9921,7 +9921,7 @@ func testLabelScopingTimestampLogic(t *testing.T, ds *Datastore) { require.Len(t, software, 0) // manual label include any - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), selfServiceInstallerID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, selfServiceInstallerID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{ label2.Name: {LabelName: label2.Name, LabelID: label2.ID}, @@ -9954,7 +9954,7 @@ func testLabelScopingTimestampLogic(t *testing.T, ds *Datastore) { require.Greater(t, label2.CreatedAt, host.LabelUpdatedAt) // Dynamic label include any - err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), selfServiceInstallerID, fleet.LabelIdentsWithScope{ + err = setOrUpdateSoftwareInstallerLabelsDB(ctx, ds.writer(ctx), ds.dialect, selfServiceInstallerID, fleet.LabelIdentsWithScope{ LabelScope: fleet.LabelScopeIncludeAny, ByName: map[string]fleet.LabelIdent{ label1.Name: {LabelName: label1.Name, LabelID: label1.ID}, From 1666fd7da20ca15742213d3f4680031d66dca164 Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Wed, 1 Apr 2026 21:18:03 -0400 Subject: [PATCH 5/6] fix(lint): address golangci-lint errors in PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - gofmt: fix formatting in mysql.go, policies.go, software.go, rebind_driver.go - unused: remove jsonObjectFunc, resolveJSONFunc (mysql.go), dialectStep (migration.go) - gosec: nolint md5 in policies.go (non-cryptographic checksum) - staticcheck S1008: simplify errors.As in postgres/errors.go - unconvert: remove unnecessary string() cast in postgres/errors.go - gocritic: use errors.New instead of fmt.Errorf in rebind_driver.go - staticcheck SA4004: remove non-looping for{} wrapper in castJsonbBuildObjectParams - testifylint: use assert.Empty in dialect_mysql_test.go, nolint goroutine requires in packs_test.go - modernize: interface{} → any in aggregated_stats.go, hosts.go, testing_utils.go - modernize: strings.SplitSeq in testing_utils.go, new(bool) in postgres_smoke_test.go - test: add mysqlDialect{} to mockDatastore to prevent nil dereference in TestGetContextTryStmt --- server/datastore/mysql/aggregated_stats.go | 4 +- server/datastore/mysql/dialect_mysql_test.go | 2 +- server/datastore/mysql/hosts.go | 8 +- .../mysql/migrations/tables/migration.go | 19 -- server/datastore/mysql/mysql.go | 17 +- server/datastore/mysql/mysql_test.go | 1 + server/datastore/mysql/packs_test.go | 4 +- server/datastore/mysql/policies.go | 4 +- server/datastore/mysql/postgres_smoke_test.go | 2 +- server/datastore/mysql/software.go | 1 - server/datastore/mysql/testing_utils.go | 4 +- server/platform/postgres/errors.go | 8 +- server/platform/postgres/rebind_driver.go | 193 +++++++++--------- 13 files changed, 113 insertions(+), 154 deletions(-) diff --git a/server/datastore/mysql/aggregated_stats.go b/server/datastore/mysql/aggregated_stats.go index 69c42314e3b..fcdd2286951 100644 --- a/server/datastore/mysql/aggregated_stats.go +++ b/server/datastore/mysql/aggregated_stats.go @@ -76,7 +76,7 @@ func getPercentileQuery(aggregate fleet.AggregatedStatsType, time string, percen } func setP50AndP95Map( - ctx context.Context, tx sqlx.QueryerContext, aggregate fleet.AggregatedStatsType, time string, id uint, statsMap map[string]interface{}, isPG bool, + ctx context.Context, tx sqlx.QueryerContext, aggregate fleet.AggregatedStatsType, time string, id uint, statsMap map[string]any, isPG bool, ) error { var p50, p95 float64 @@ -118,7 +118,7 @@ func (ds *Datastore) CalculateAggregatedPerfStatsPercentiles(ctx context.Context // We are using the reader because the below SELECT queries are expensive, and we don't want to impact the performance of the writer. reader := ds.reader(ctx) var totalExecutions int - statsMap := make(map[string]interface{}) + statsMap := make(map[string]any) // many queries is not ideal, but getting both values and totals in the same query was a bit more complicated // so I went for the simpler approach first, we can optimize later diff --git a/server/datastore/mysql/dialect_mysql_test.go b/server/datastore/mysql/dialect_mysql_test.go index b140c044ebc..2af06684d76 100644 --- a/server/datastore/mysql/dialect_mysql_test.go +++ b/server/datastore/mysql/dialect_mysql_test.go @@ -23,7 +23,7 @@ func TestMysqlDialectSQL(t *testing.T) { }) t.Run("OnConflictDoNothing", func(t *testing.T) { - assert.Equal(t, "", d.OnConflictDoNothing("id")) + assert.Empty(t, d.OnConflictDoNothing("id")) }) t.Run("GroupConcat", func(t *testing.T) { diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 4df51496b79..2a193588e30 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -297,7 +297,7 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, dialect DialectHelper seen[key] = i } if len(seen) < scheduledQueriesQueryCount { - var dedupedArgs []interface{} + var dedupedArgs []any dedupedCount := 0 for i := 0; i < scheduledQueriesQueryCount; i++ { base := i * argsPerRow @@ -332,7 +332,7 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, dialect DialectHelper seen[key] = i } if len(seen) < userPacksQueryCount { - var dedupedArgs []interface{} + var dedupedArgs []any dedupedCount := 0 for i := 0; i < userPacksQueryCount; i++ { base := i * argsPerRow @@ -358,7 +358,7 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, dialect DialectHelper // which avoids NOT NULL violations on PG). argsPerRow := 12 // 2 (subquery: teamID, name) + 10 (values) var selectParts []string - var reorderedArgs []interface{} + var reorderedArgs []any for i := 0; i < scheduledQueriesQueryCount; i++ { base := i * argsPerRow selectParts = append(selectParts, @@ -441,7 +441,7 @@ func saveHostPackStatsDB(ctx context.Context, db *sqlx.DB, dialect DialectHelper // a second time" when multiple scheduled queries reference the same query_id. argsPerRow := 12 // 2 (subquery: packName, sqName) + 10 (values) var selectParts []string - var reorderedArgs []interface{} + var reorderedArgs []any for i := 0; i < userPacksQueryCount; i++ { base := i * argsPerRow selectParts = append(selectParts, diff --git a/server/datastore/mysql/migrations/tables/migration.go b/server/datastore/mysql/migrations/tables/migration.go index 8a5791ec15d..e6eaa33ee82 100644 --- a/server/datastore/mysql/migrations/tables/migration.go +++ b/server/datastore/mysql/migrations/tables/migration.go @@ -113,25 +113,6 @@ func withSteps(steps []migrationStep, tx *sql.Tx) error { return nil } -// dialectStep returns a migrationStep that executes the MySQL statement -// for MySQL databases and the PostgreSQL statement for PostgreSQL databases. -// The driver is determined at migration time from the goose client's dialect. -// Pass an empty string for either statement to make it a no-op for that dialect. -func dialectStep(mysqlStmt, pgStmt string) migrationStep { - return func(tx *sql.Tx) error { - // Determine dialect from the connection's driver. - // In the current architecture, the migration client always uses MySqlDialect, - // so pgStmt will not be executed until a PostgreSQL migration client exists. - // For now, always execute mysqlStmt. - stmt := mysqlStmt - if stmt == "" { - return nil - } - _, err := tx.Exec(stmt) - return err - } -} - // migrationHelper provides dialect-specific schema introspection for migrations. // The default implementation uses MySQL information_schema. // When PostgreSQL support is added, a pgMigrationHelper will use pg_catalog. diff --git a/server/datastore/mysql/mysql.go b/server/datastore/mysql/mysql.go index 230a7d7af8d..fb581b6cdaa 100644 --- a/server/datastore/mysql/mysql.go +++ b/server/datastore/mysql/mysql.go @@ -33,8 +33,8 @@ import ( nano_push "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/push" scep_depot "github.com/fleetdm/fleet/v4/server/mdm/scep/depot" common_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql" - "github.com/go-sql-driver/mysql" _ "github.com/fleetdm/fleet/v4/server/platform/postgres" // register pgx-rebind driver for PostgreSQL + "github.com/go-sql-driver/mysql" "github.com/hashicorp/go-multierror" "github.com/jmoiron/sqlx" "go.opentelemetry.io/otel/attribute" @@ -1567,18 +1567,3 @@ func batchProcessDB[T any]( } return nil } - -// jsonObjectFunc returns the SQL function name for building JSON objects. -// MySQL: JSON_OBJECT, PostgreSQL: jsonb_build_object -func (ds *Datastore) jsonObjectFunc() string { - if ds.dialect.ReturningID() != "" { - return "jsonb_build_object" - } - return "JSON_OBJECT" -} - -// resolveJSONFunc replaces %JSON_OBJECT% placeholder with the dialect-specific -// JSON object builder function name. -func (ds *Datastore) resolveJSONFunc(sql string) string { - return strings.ReplaceAll(sql, "%JSON_OBJECT%", ds.jsonObjectFunc()) -} diff --git a/server/datastore/mysql/mysql_test.go b/server/datastore/mysql/mysql_test.go index b9198341f2d..90123d583b7 100644 --- a/server/datastore/mysql/mysql_test.go +++ b/server/datastore/mysql/mysql_test.go @@ -249,6 +249,7 @@ func mockDatastore(t *testing.T) (sqlmock.Sqlmock, *Datastore) { primary: dbmock, replica: dbmock, logger: slog.New(slog.DiscardHandler), + dialect: mysqlDialect{}, } return mock, ds diff --git a/server/datastore/mysql/packs_test.go b/server/datastore/mysql/packs_test.go index d02d1370c08..81af9b3d0cb 100644 --- a/server/datastore/mysql/packs_test.go +++ b/server/datastore/mysql/packs_test.go @@ -498,7 +498,7 @@ func testPacksApplyStatsNotLocking(t *testing.T, ds *Datastore) { require.NoError(t, err) amount := rand.Intn(5000) - require.NoError(t, saveHostPackStatsDB(context.Background(), ds.writer(context.Background()), ds.dialect, host.TeamID, host.ID, randomPackStatsForHost(pack.ID, pack.Name, *pack.Type, schedQueries, amount))) + require.NoError(t, saveHostPackStatsDB(context.Background(), ds.writer(context.Background()), ds.dialect, host.TeamID, host.ID, randomPackStatsForHost(pack.ID, pack.Name, *pack.Type, schedQueries, amount))) //nolint:testifylint // require in goroutine is intentional for this stress test } } }() @@ -550,7 +550,7 @@ func testPacksApplyStatsNotLockingTryTwo(t *testing.T, ds *Datastore) { require.NoError(t, err) amount := rand.Intn(5000) - require.NoError(t, saveHostPackStatsDB(context.Background(), ds.writer(context.Background()), ds.dialect, host.TeamID, host.ID, randomPackStatsForHost(pack.ID, pack.Name, *pack.Type, schedQueries, amount))) + require.NoError(t, saveHostPackStatsDB(context.Background(), ds.writer(context.Background()), ds.dialect, host.TeamID, host.ID, randomPackStatsForHost(pack.ID, pack.Name, *pack.Type, schedQueries, amount))) //nolint:testifylint // require in goroutine is intentional for this stress test } } }() diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index 982736985ac..76214e79fba 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -275,13 +275,13 @@ func policiesChecksumComputedColumn() string { } // policyChecksum computes the checksum for a policy in Go (portable across databases). -// The checksum is MD5(CONCAT_WS(\x00, COALESCE(team_id, ''), name)) as raw bytes. +// The checksum is MD5(CONCAT_WS(\x00, COALESCE(team_id, ”), name)) as raw bytes. func policyChecksum(teamID *uint, name string) []byte { var teamStr string if teamID != nil { teamStr = fmt.Sprintf("%d", *teamID) } - h := md5.Sum([]byte(teamStr + "\x00" + name)) + h := md5.Sum([]byte(teamStr + "\x00" + name)) //nolint:gosec // MD5 used for non-cryptographic checksum return h[:] } diff --git a/server/datastore/mysql/postgres_smoke_test.go b/server/datastore/mysql/postgres_smoke_test.go index 8a3651f596e..60fc513bf71 100644 --- a/server/datastore/mysql/postgres_smoke_test.go +++ b/server/datastore/mysql/postgres_smoke_test.go @@ -379,7 +379,7 @@ func TestPostgresDatastoreOperations(t *testing.T) { // --- Host disk encryption key --- t.Run("SetOrUpdateHostDiskEncryptionKey", func(t *testing.T) { - _, err := ds.SetOrUpdateHostDiskEncryptionKey(ctx, host, "test-key", "test-client", ptr.Bool(false)) + _, err := ds.SetOrUpdateHostDiskEncryptionKey(ctx, host, "test-key", "test-client", new(bool)) if err != nil { t.Logf("FAIL SetOrUpdateHostDiskEncryptionKey: %v", err) } diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index dcfc7d484bf..9f15851b25b 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -6724,4 +6724,3 @@ func (ds *Datastore) SoftwareLiteByID( return results, nil } - diff --git a/server/datastore/mysql/testing_utils.go b/server/datastore/mysql/testing_utils.go index 6cd3c594c50..6e71cb76f77 100644 --- a/server/datastore/mysql/testing_utils.go +++ b/server/datastore/mysql/testing_utils.go @@ -538,7 +538,7 @@ func CreatePostgresDS(t *testing.T) *Datastore { var stmts []string inDollarQuote := false var current strings.Builder - for _, line := range strings.Split(schema, "\n") { + for line := range strings.SplitSeq(schema, "\n") { trimmed := strings.TrimSpace(line) // Count $$ occurrences — odd count toggles dollar-quote state if strings.Count(trimmed, "$$")%2 == 1 { @@ -652,7 +652,7 @@ func ExecAdhocSQLWithError(ds *Datastore, fn func(q sqlx.ExtContext) error) erro // InsertAndGetLastID executes an INSERT statement and returns the auto-generated ID. // On MySQL it uses LastInsertId(); on PG it appends RETURNING id and scans the result. -func InsertAndGetLastID(ctx context.Context, ds *Datastore, query string, args ...interface{}) (int64, error) { +func InsertAndGetLastID(ctx context.Context, ds *Datastore, query string, args ...any) (int64, error) { if ds.dialect.IsPostgres() { pgQuery := query + " RETURNING id" var id int64 diff --git a/server/platform/postgres/errors.go b/server/platform/postgres/errors.go index d7bf6969f14..ab2b518ca70 100644 --- a/server/platform/postgres/errors.go +++ b/server/platform/postgres/errors.go @@ -76,11 +76,7 @@ func IsBadConnection(err error) bool { } var netErr *net.OpError - if errors.As(err, &netErr) { - return true - } - - return false + return errors.As(err, &netErr) } // hasErrorCode checks if the error (or any wrapped error) contains the given @@ -97,7 +93,7 @@ func hasErrorCode(err error, code string) bool { } var pgxErr pgxError if errors.As(err, &pgxErr) { - return string(pgxErr.Code()) == code + return pgxErr.Code() == code } // Check for lib/pq-style error (has Code field via the pq.Error type). diff --git a/server/platform/postgres/rebind_driver.go b/server/platform/postgres/rebind_driver.go index eab5e690827..b9db0dfe74b 100644 --- a/server/platform/postgres/rebind_driver.go +++ b/server/platform/postgres/rebind_driver.go @@ -9,6 +9,7 @@ import ( "context" "database/sql" "database/sql/driver" + "errors" "fmt" "regexp" "strings" @@ -20,25 +21,25 @@ import ( // Pre-compiled regexes used in rebindQuery to avoid per-query compilation overhead. var ( - reUUIDBinUpper = regexp.MustCompile(`UUID_TO_BIN\(UUID\(\),\s*true\)`) - reUUIDBinLower = regexp.MustCompile(`UUID_TO_BIN\(uuid\(\),\s*true\)`) - reUUIDBinTrue = regexp.MustCompile(`UUID_TO_BIN\(([^,)]+),\s*true\)`) - reUUIDBin = regexp.MustCompile(`UUID_TO_BIN\(([^,)]+)\)`) - reUUID = regexp.MustCompile(`(?i)\bUUID\(\)`) - reBinToUUIDTrue = regexp.MustCompile(`BIN_TO_UUID\(([^,)]+),\s*true\)`) - reBinToUUID = regexp.MustCompile(`BIN_TO_UUID\(([^,)]+)\)`) - reTimeDiff = regexp.MustCompile(`TIMEDIFF\(([^,]+),\s*([^)]+)\)`) - reTimeToSec = regexp.MustCompile(`TIME_TO_SEC\(([^)]+)\)`) - reFromDual = regexp.MustCompile(`(?i)\s+FROM\s+DUAL\b`) - reSeparator = regexp.MustCompile(`(?i)\bSEPARATOR\s+'([^']*)'`) - reTimestamp = regexp.MustCompile(`\bTIMESTAMP\(([^)]+)\)`) - reMaxDenylisted = regexp.MustCompile(`MAX\(([^)]*\.denylisted)\)`) + reUUIDBinUpper = regexp.MustCompile(`UUID_TO_BIN\(UUID\(\),\s*true\)`) + reUUIDBinLower = regexp.MustCompile(`UUID_TO_BIN\(uuid\(\),\s*true\)`) + reUUIDBinTrue = regexp.MustCompile(`UUID_TO_BIN\(([^,)]+),\s*true\)`) + reUUIDBin = regexp.MustCompile(`UUID_TO_BIN\(([^,)]+)\)`) + reUUID = regexp.MustCompile(`(?i)\bUUID\(\)`) + reBinToUUIDTrue = regexp.MustCompile(`BIN_TO_UUID\(([^,)]+),\s*true\)`) + reBinToUUID = regexp.MustCompile(`BIN_TO_UUID\(([^,)]+)\)`) + reTimeDiff = regexp.MustCompile(`TIMEDIFF\(([^,]+),\s*([^)]+)\)`) + reTimeToSec = regexp.MustCompile(`TIME_TO_SEC\(([^)]+)\)`) + reFromDual = regexp.MustCompile(`(?i)\s+FROM\s+DUAL\b`) + reSeparator = regexp.MustCompile(`(?i)\bSEPARATOR\s+'([^']*)'`) + reTimestamp = regexp.MustCompile(`\bTIMESTAMP\(([^)]+)\)`) + reMaxDenylisted = regexp.MustCompile(`MAX\(([^)]*\.denylisted)\)`) // MAX(prof_*) columns from boolean subqueries (android/apple MDM profile status aggregation) - reMaxBooleanCols = regexp.MustCompile(`MAX\(((?:prof|fv|rl|decl)_(?:pending|failed|verifying|verified)|android_prof_(?:pending|failed|verifying|verified))\)`) - reLimitTrailing = regexp.MustCompile(`(?i)\s+LIMIT\s+\d+\s*$`) - reJSONExtractFunc = regexp.MustCompile(`JSON_EXTRACT\((\w+),\s*(\?|'[^']*')\)`) - reJSONPath = regexp.MustCompile(`->>?'\$\.[^']*'`) - reTimestampDiff = regexp.MustCompile(`(?i)TIMESTAMPDIFF\(\s*SECOND\s*,\s*(.+?)\s*,\s*(.+?)\s*\)`) + reMaxBooleanCols = regexp.MustCompile(`MAX\(((?:prof|fv|rl|decl)_(?:pending|failed|verifying|verified)|android_prof_(?:pending|failed|verifying|verified))\)`) + reLimitTrailing = regexp.MustCompile(`(?i)\s+LIMIT\s+\d+\s*$`) + reJSONExtractFunc = regexp.MustCompile(`JSON_EXTRACT\((\w+),\s*(\?|'[^']*')\)`) + reJSONPath = regexp.MustCompile(`->>?'\$\.[^']*'`) + reTimestampDiff = regexp.MustCompile(`(?i)TIMESTAMPDIFF\(\s*SECOND\s*,\s*(.+?)\s*,\s*(.+?)\s*\)`) reNormalizeDuplicateKey = regexp.MustCompile(`(?i)ON\s+DUPLICATE\s+KEY\s+UPDATE`) // MySQL: INSERT INTO table () VALUES () — empty column/value lists for auto-increment-only inserts reEmptyValues = regexp.MustCompile(`(?i)(INSERT\s+INTO\s+\S+\s+)\(\s*\)\s*VALUES\s*\(\s*\)`) @@ -483,7 +484,7 @@ func (r *rebindRows) NextResultSet() error { if rs, ok := r.Rows.(driver.RowsNextResultSet); ok { return rs.NextResultSet() } - return fmt.Errorf("not supported") + return errors.New("not supported") } // coerceBoolArgsForTextCast converts Go bool args to "true"/"false" strings @@ -868,65 +869,62 @@ func isIdentChar(c byte) bool { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' } - // castJsonbBuildObjectParams adds ::text casts to ? placeholders inside jsonb_build_object() calls. // PG's jsonb_build_object has a VARIADIC "any" signature, so it can't infer placeholder parameter types. // Casting to ::text makes all JSON values strings, which is compatible with ->>' text extraction. // Handles nested jsonb_build_object and subqueries via paren-balancing. func castJsonbBuildObjectParams(query string) string { const prefix = "jsonb_build_object(" - for { - idx := strings.Index(query, prefix) - if idx < 0 { - return query - } - start := idx + len(prefix) - depth := 1 - i := start - // Walk through the jsonb_build_object args, adding ::text to ? placeholders - // in ALL positions (both keys and values). PG's jsonb_build_object has a - // VARIADIC "any" signature, so it can't infer any placeholder parameter types. - var result strings.Builder - result.WriteString(query[:start]) - argStart := i + idx := strings.Index(query, prefix) + if idx < 0 { + return query + } + start := idx + len(prefix) + depth := 1 + i := start + // Walk through the jsonb_build_object args, adding ::text to ? placeholders + // in ALL positions (both keys and values). PG's jsonb_build_object has a + // VARIADIC "any" signature, so it can't infer any placeholder parameter types. + var result strings.Builder + result.WriteString(query[:start]) + argStart := i - for i < len(query) && depth > 0 { - switch query[i] { - case '(': - depth++ - i++ - case ')': - depth-- - if depth == 0 { - // Process the last argument - arg := query[argStart:i] - arg = castPlaceholdersInArg(arg) - result.WriteString(arg) - result.WriteByte(')') - } + for i < len(query) && depth > 0 { + switch query[i] { + case '(': + depth++ + i++ + case ')': + depth-- + if depth == 0 { + // Process the last argument + arg := query[argStart:i] + arg = castPlaceholdersInArg(arg) + result.WriteString(arg) + result.WriteByte(')') + } + i++ + case ',': + if depth == 1 { + arg := query[argStart:i] + arg = castPlaceholdersInArg(arg) + result.WriteString(arg) + result.WriteByte(',') + argStart = i + 1 i++ - case ',': - if depth == 1 { - arg := query[argStart:i] - arg = castPlaceholdersInArg(arg) - result.WriteString(arg) - result.WriteByte(',') - argStart = i + 1 - i++ - } else { - i++ - } - default: + } else { i++ } + default: + i++ } - if depth != 0 { - return query // unbalanced, leave as-is - } - // Recursively process the rest of the query - result.WriteString(castJsonbBuildObjectParams(query[i:])) - return result.String() } + if depth != 0 { + return query // unbalanced, leave as-is + } + // Recursively process the rest of the query + result.WriteString(castJsonbBuildObjectParams(query[i:])) + return result.String() } // castPlaceholdersInArg adds ::text to bare ? placeholders in a jsonb_build_object value argument. @@ -1000,9 +998,9 @@ func rewriteJSONPath(query string) string { if len(parts) == 1 { // Simple case: no dots if isText { - return "->>'"+parts[0]+"'" + return "->>'" + parts[0] + "'" } - return "->'"+parts[0]+"'" + return "->'" + parts[0] + "'" } // Multi-level path: all but last use ->, last uses the original operator var sb strings.Builder @@ -1381,36 +1379,36 @@ func splitTopLevel(s string, delim byte) []string { // This handles cases not going through the dialect helper. // knownPrimaryKeys maps table names to their primary key columns for ON CONFLICT resolution. var knownPrimaryKeys = map[string]string{ - "host_dep_assignments": "host_id", - "host_mdm_idp_accounts": "host_uuid", - "host_mdm_apple_declarations": "host_uuid,declaration_uuid", - "mdm_declaration_labels": "apple_declaration_uuid,label_name", - "scim_user_group": "scim_user_id,group_id", - "host_munki_issues": "host_id,munki_issue_id", - "host_munki_info": "host_id", - "cron_stats": "id", - "nano_command_results": "id,command_uuid", - "host_mdm_apple_bootstrap_packages": "host_uuid", - "mdm_configuration_profile_labels": "id", - "app_config_json": "id", - "host_mdm_android_profiles": "host_uuid,profile_uuid", - "host_conditional_access": "host_id", - "host_mdm": "host_id", - "host_display_names": "host_id", - "host_emails": "id", - "label_membership": "host_id,label_id", - "host_software": "host_id,software_id", - "software_host_counts": "software_id,team_id", - "nano_enrollment_queue": "id,command_uuid", - "host_mdm_windows_profiles": "host_uuid,profile_uuid", + "host_dep_assignments": "host_id", + "host_mdm_idp_accounts": "host_uuid", + "host_mdm_apple_declarations": "host_uuid,declaration_uuid", + "mdm_declaration_labels": "apple_declaration_uuid,label_name", + "scim_user_group": "scim_user_id,group_id", + "host_munki_issues": "host_id,munki_issue_id", + "host_munki_info": "host_id", + "cron_stats": "id", + "nano_command_results": "id,command_uuid", + "host_mdm_apple_bootstrap_packages": "host_uuid", + "mdm_configuration_profile_labels": "id", + "app_config_json": "id", + "host_mdm_android_profiles": "host_uuid,profile_uuid", + "host_conditional_access": "host_id", + "host_mdm": "host_id", + "host_display_names": "host_id", + "host_emails": "id", + "label_membership": "host_id,label_id", + "host_software": "host_id,software_id", + "software_host_counts": "software_id,team_id", + "nano_enrollment_queue": "id,command_uuid", + "host_mdm_windows_profiles": "host_uuid,profile_uuid", // NanoMDM/NanoDEP tables - "nano_dep_names": "name", - "nano_devices": "id", - "nano_users": "id,device_id", - "nano_enrollments": "id", - "nano_cert_auth_associations": "id,sha256", - "nano_push_certs": "topic", - "host_certificate_templates": "host_uuid,certificate_template_id", + "nano_dep_names": "name", + "nano_devices": "id", + "nano_users": "id,device_id", + "nano_enrollments": "id", + "nano_cert_auth_associations": "id,sha256", + "nano_push_certs": "topic", + "host_certificate_templates": "host_uuid,certificate_template_id", } func rewriteOnDuplicateKey(query string) string { @@ -1587,4 +1585,3 @@ func rewriteUpdateJoin(query string) string { return fmt.Sprintf("UPDATE %s SET %s FROM %s WHERE %s", table1, setClause, strings.Join(fromTables, ", "), allConditions) } - From 3abf07e983ce6a0d098f4794732ca52e5cb8f663 Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Wed, 1 Apr 2026 22:42:44 -0400 Subject: [PATCH 6/6] fix(lint): suppress gosec G501 on crypto/md5 import in policies.go --- server/datastore/mysql/policies.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index 76214e79fba..50af5118eac 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -2,7 +2,7 @@ package mysql import ( "context" - "crypto/md5" + "crypto/md5" //nolint:gosec // MD5 used for non-cryptographic checksum only "database/sql" "encoding/json" "errors"