feat: sync virtual user files to a project's Git repo

admin/admin.go

@@ -59,6 +59,7 @@ type Service struct {
 	StoppedDeploymentRetention time.Duration
 	Biller                     billing.Biller
 	PaymentProvider            payment.Provider
+	GitWriter                  *GitWriter
 }
 
 func New(ctx context.Context, opts *Options, logger *zap.Logger, issuer *auth.Issuer, emailClient *email.Client, github Github, aiService drivers.AIService, assets *storage.BucketHandle, biller billing.Biller, p payment.Provider) (*Service, error) {
@@ -144,6 +145,7 @@ func New(ctx context.Context, opts *Options, logger *zap.Logger, issuer *auth.Is
 		StoppedDeploymentRetention: opts.StoppedDeploymentRetention,
 		Biller:                     biller,
 		PaymentProvider:            p,
+		GitWriter:                  NewGitWriter(),
 	}, nil
 }
 

admin/database/database.go

@@ -91,6 +91,17 @@ type DB interface {
 	InsertProject(ctx context.Context, opts *InsertProjectOptions) (*Project, error)
 	DeleteProject(ctx context.Context, id string) error
 	UpdateProject(ctx context.Context, id string, opts *UpdateProjectOptions) (*Project, error)
+	// UpdateProjectUserFilesSyncInterval sets the interval for periodic auto-sync of user files to Git.
+	// Zero disables auto-sync.
+	UpdateProjectUserFilesSyncInterval(ctx context.Context, id string, seconds int64) error
+	// UpdateProjectUserFilesSyncedOn records that a user-files sync completed successfully just now,
+	// along with any warning it produced (e.g. direct Git edits it overwrote).
+	// It clears the error and warning from previous attempts.
+	UpdateProjectUserFilesSyncedOn(ctx context.Context, id, syncWarning string) error
+	// UpdateProjectUserFilesSyncError records the outcome of a failed user-files sync attempt,
+	// clearing any warning from a previous attempt. Pass an empty string to clear both
+	// (e.g. when a new sync is requested).
+	UpdateProjectUserFilesSyncError(ctx context.Context, id, syncError string) error
 	CountProjectsForOrganization(ctx context.Context, orgID string) (int, error)
 	CountProjectsQuotaUsage(ctx context.Context, orgID string) (*ProjectsQuotaUsage, error)
 	FindProjectWhitelistedDomain(ctx context.Context, projectID, domain string) (*ProjectWhitelistedDomain, error)
@@ -310,10 +321,27 @@ type DB interface {
 
 	FindVirtualFiles(ctx context.Context, projectID, environment string, afterUpdatedOn time.Time, afterPath string, limit int) ([]*VirtualFile, error)
 	FindVirtualFile(ctx context.Context, projectID, environment, path string) (*VirtualFile, error)
+	// FindVirtualFileByName finds a live (not soft-deleted) virtual file by its resource name (the path stem)
+	// and subdir, independent of the owner segment in its path. It matches both the new
+	// "user_files/<seg>/<subdir>/<name>.yaml" layout and the legacy flat layout ("alerts/", "reports/",
+	// "personal/") so lookups keep working across the migration. Deleted files return ErrNotFound.
+	FindVirtualFileByName(ctx context.Context, projectID, environment, subdir, name string) (*VirtualFile, error)
 	// FindVirtualFilesByOwner TODO: pagination
 	FindVirtualFilesByOwner(ctx context.Context, projectID, environment, ownerID string) ([]*VirtualFile, error)
+	// FindStagedVirtualFiles returns the staging buffer for a project: every virtual file (including
+	// soft-deleted tombstones) whose content has not yet been confirmed in Git (sync_state != 'synced').
+	FindStagedVirtualFiles(ctx context.Context, projectID, environment string) ([]*VirtualFile, error)
+	// CountStagedVirtualFiles counts the files FindStagedVirtualFiles would return.
+	CountStagedVirtualFiles(ctx context.Context, projectID, environment string) (int, error)
+	// FindProjectIDsForUserFilesAutoSync returns the IDs of Git-connected projects that have auto-sync
+	// enabled, are due for a sync, and have staged virtual files. It's used by the periodic sync sweep.
+	FindProjectIDsForUserFilesAutoSync(ctx context.Context) ([]string, error)
 	UpsertVirtualFile(ctx context.Context, opts *InsertVirtualFileOptions) error
 	UpdateVirtualFileDeleted(ctx context.Context, projectID, environment, path string) error
+	// MarkVirtualFileSynced marks a staged file as confirmed on the served branch, but only if its data and
+	// deleted flag still match the flushed values (i.e. it has not been edited since the flush). It bumps
+	// updated_on so the change propagates to runtimes via FindVirtualFiles. Returns true if updated.
+	MarkVirtualFileSynced(ctx context.Context, projectID, environment, path string, expectedData []byte, expectedDeleted bool) (bool, error)
 	DeleteExpiredVirtualFiles(ctx context.Context, retention time.Duration) error
 
 	FindAsset(ctx context.Context, id string) (*Asset, error)
@@ -493,6 +521,17 @@ type Project struct {
 	// Subpath is an optional subpath for the project files within the Git repository.
 	// It enables Rill files to be stored in a monorepo.
 	Subpath string `db:"subpath"`
+	// UserFilesSyncIntervalSeconds is the interval for periodic auto-sync of user files to Git.
+	// Zero means auto-sync is disabled.
+	UserFilesSyncIntervalSeconds int64 `db:"user_files_sync_interval_seconds"`
+	// UserFilesSyncedOn is the time of the last successful user-files sync (manual or scheduled).
+	UserFilesSyncedOn *time.Time `db:"user_files_synced_on"`
+	// UserFilesSyncError is the error from the most recent user-files sync attempt.
+	// It is empty when the last sync succeeded (or none ran yet).
+	UserFilesSyncError string `db:"user_files_sync_error"`
+	// UserFilesSyncWarning is the warning from the most recent successful user-files sync,
+	// e.g. direct Git edits the sync overwrote. It is empty when the last sync was clean.
+	UserFilesSyncWarning string `db:"user_files_sync_warning"`
 	// ProdVersion is the runtime version to use for the production deployment.
 	ProdVersion string `db:"prod_version"`
 	// PrimaryBranch is the Git branch to use for the primary production deployment for Git-connected projects.
@@ -1227,15 +1266,30 @@ type UpdateBookmarkOptions struct {
 	Shared      bool   `json:"shared"`
 }
 
-// VirtualFile represents an ad-hoc file for a project (not managed in Git)
+// VirtualFile represents an ad-hoc file for a project.
+// Virtual files act as a staging buffer for files that are promoted into the project's Git repo.
 type VirtualFile struct {
 	Path      string    `db:"path"`
 	Data      []byte    `db:"data"`
 	OwnerID   *string   `db:"owner_id"`
 	Deleted   bool      `db:"deleted"`
 	UpdatedOn time.Time `db:"updated_on"`
+	// SyncState tracks where the file is in the Git promotion lifecycle: "dirty" or "synced".
+	// Synced rows are kept as a path index for later edits; only synced tombstones are eventually purged.
+	SyncState string `db:"sync_state"`
+	// BaseData is the file's content when it last transitioned from synced to dirty, i.e. the version the
+	// staged change was based on. It is nil for rows that were never synced, and only populated by
+	// FindStagedVirtualFiles (other queries skip it to keep the runtime feed payload small).
+	BaseData []byte `db:"base_data"`
 }
 
+// Git sync states for a VirtualFile.
+// "pending_pr" is reserved for a future pull request flow for protected branches.
+const (
+	VirtualFileSyncStateDirty  = "dirty"
+	VirtualFileSyncStateSynced = "synced"
+)
+
 // InsertVirtualFileOptions defines options for inserting a VirtualFile
 type InsertVirtualFileOptions struct {
 	ProjectID   string

admin/database/postgres/migrations/0096.sql

@@ -0,0 +1,23 @@
+-- Track the Git-sync state of each virtual file.
+-- Virtual files act as a staging buffer: writes land here instantly and are later
+-- promoted into the project's Git repo under user_files/<segment>/<kind>/<name>.yaml.
+-- sync_state: 'dirty' (staged, not yet in Git) or 'synced' (content confirmed on the served branch;
+-- the row is kept as a path index for later edits). 'pending_pr' is reserved for a future pull request flow.
+ALTER TABLE virtual_files ADD COLUMN sync_state TEXT NOT NULL DEFAULT 'dirty';
+
+-- Snapshot of the file's content when it last transitioned from synced to dirty, i.e. the version the
+-- staged change was based on. NULL for rows that were never synced. The sync job compares it to the
+-- Git copy to detect (and report) when a sync overwrites edits made directly in Git while the row was dirty.
+ALTER TABLE virtual_files ADD COLUMN base_data BYTEA;
+
+-- Cheap lookups of a project's staged (unsynced) files for the status API and the periodic sync sweep.
+CREATE INDEX virtual_files_staged_idx ON virtual_files (project_id, environment) WHERE sync_state <> 'synced';
+
+-- Interval for periodic auto-sync of user files to Git. 0 = disabled (default). Configured by project admins.
+ALTER TABLE projects ADD COLUMN user_files_sync_interval_seconds BIGINT NOT NULL DEFAULT 0;
+-- Time of the last successful user-files sync (manual or scheduled). Used to compute when auto-sync is due.
+ALTER TABLE projects ADD COLUMN user_files_synced_on TIMESTAMPTZ;
+-- Error from the most recent user-files sync attempt. Empty when the last sync succeeded (or none ran yet).
+ALTER TABLE projects ADD COLUMN user_files_sync_error TEXT NOT NULL DEFAULT '';
+-- Warning from the most recent successful user-files sync, e.g. direct Git edits that the sync overwrote.
+ALTER TABLE projects ADD COLUMN user_files_sync_warning TEXT NOT NULL DEFAULT '';

admin/database/postgres/postgres.go

@@ -565,6 +565,21 @@ func (c *connection) UpdateProject(ctx context.Context, id string, opts *databas
 	return c.projectFromDTO(res)
 }
 
+func (c *connection) UpdateProjectUserFilesSyncInterval(ctx context.Context, id string, seconds int64) error {
+	res, err := c.getDB(ctx).ExecContext(ctx, "UPDATE projects SET user_files_sync_interval_seconds = $1, updated_on = now() WHERE id = $2", seconds, id)
+	return checkUpdateRow("project", res, err)
+}
+
+func (c *connection) UpdateProjectUserFilesSyncedOn(ctx context.Context, id, syncWarning string) error {
+	res, err := c.getDB(ctx).ExecContext(ctx, "UPDATE projects SET user_files_synced_on = now(), user_files_sync_error = '', user_files_sync_warning = $1 WHERE id = $2", syncWarning, id)
+	return checkUpdateRow("project", res, err)
+}
+
+func (c *connection) UpdateProjectUserFilesSyncError(ctx context.Context, id, syncError string) error {
+	res, err := c.getDB(ctx).ExecContext(ctx, "UPDATE projects SET user_files_sync_error = $1, user_files_sync_warning = '' WHERE id = $2", syncError, id)
+	return checkUpdateRow("project", res, err)
+}
+
 func (c *connection) CountProjectsQuotaUsage(ctx context.Context, orgID string) (*database.ProjectsQuotaUsage, error) {
 	res := &database.ProjectsQuotaUsage{}
 	err := c.getDB(ctx).QueryRowxContext(ctx, `
@@ -2894,7 +2909,7 @@ func (c *connection) DeleteBookmark(ctx context.Context, bookmarkID string) erro
 func (c *connection) FindVirtualFiles(ctx context.Context, projectID, environment string, afterUpdatedOn time.Time, afterPath string, limit int) ([]*database.VirtualFile, error) {
 	var res []*database.VirtualFile
 	err := c.getDB(ctx).SelectContext(ctx, &res, `
-		SELECT path, data, owner_id, deleted, updated_on
+		SELECT path, data, owner_id, deleted, updated_on, sync_state
 		FROM virtual_files
 		WHERE project_id=$1 AND environment=$2 AND (updated_on>$3 OR updated_on=$3 AND path>$4)
 		ORDER BY updated_on, path LIMIT $5
@@ -2908,7 +2923,7 @@ func (c *connection) FindVirtualFiles(ctx context.Context, projectID, environmen
 func (c *connection) FindVirtualFile(ctx context.Context, projectID, environment, path string) (*database.VirtualFile, error) {
 	res := &database.VirtualFile{}
 	err := c.getDB(ctx).QueryRowxContext(ctx, `
-		SELECT path, data, owner_id, deleted, updated_on
+		SELECT path, data, owner_id, deleted, updated_on, sync_state
 		FROM virtual_files
 		WHERE project_id=$1 AND environment=$2 AND path=$3
 	`, projectID, environment, path).StructScan(res)
@@ -2918,10 +2933,37 @@ func (c *connection) FindVirtualFile(ctx context.Context, projectID, environment
 	return res, nil
 }
 
+func (c *connection) FindVirtualFileByName(ctx context.Context, projectID, environment, subdir, name string) (*database.VirtualFile, error) {
+	res := &database.VirtualFile{}
+	// $3 matches the legacy flat path; $4 matches the "user_files/<seg>/<subdir>/<name>.yaml" layout.
+	// Soft-deleted rows (tombstones) are excluded: a deleted resource must resolve as not found,
+	// not resurface through a name lookup. Resource names embed a random suffix, so live-row collisions
+	// are not expected.
+	legacySubdir := subdir
+	if subdir == "canvas" {
+		legacySubdir = "personal" // Personal files were stored under "personal/" before the user_files layout.
+	}
+	legacyPath := legacySubdir + "/" + name + ".yaml"
+	// The name is caller-provided, so escape LIKE metacharacters: a stray % or _ must match literally,
+	// not act as a wildcard that could resolve to a different file.
+	escapedName := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(name)
+	newPattern := "user_files/%/" + subdir + "/" + escapedName + ".yaml"
+	err := c.getDB(ctx).QueryRowxContext(ctx, `
+		SELECT path, data, owner_id, deleted, updated_on, sync_state
+		FROM virtual_files
+		WHERE project_id=$1 AND environment=$2 AND deleted=FALSE AND (path=$3 OR path LIKE $4)
+		LIMIT 1
+	`, projectID, environment, legacyPath, newPattern).StructScan(res)
+	if err != nil {
+		return nil, parseErr("virtual file", err)
+	}
+	return res, nil
+}
+
 func (c *connection) FindVirtualFilesByOwner(ctx context.Context, projectID, environment, ownerID string) ([]*database.VirtualFile, error) {
 	var res []*database.VirtualFile
 	err := c.getDB(ctx).SelectContext(ctx, &res, `
-		SELECT path, data, owner_id, deleted, updated_on
+		SELECT path, data, owner_id, deleted, updated_on, sync_state
 		FROM virtual_files
 		WHERE project_id=$1 AND environment=$2 AND deleted=FALSE AND owner_id=$3
 		ORDER BY path
@@ -2932,18 +2974,68 @@ func (c *connection) FindVirtualFilesByOwner(ctx context.Context, projectID, env
 	return res, nil
 }
 
+func (c *connection) FindStagedVirtualFiles(ctx context.Context, projectID, environment string) ([]*database.VirtualFile, error) {
+	var res []*database.VirtualFile
+	err := c.getDB(ctx).SelectContext(ctx, &res, `
+		SELECT path, data, owner_id, deleted, updated_on, sync_state, base_data
+		FROM virtual_files
+		WHERE project_id=$1 AND environment=$2 AND sync_state != 'synced'
+		ORDER BY path
+	`, projectID, environment)
+	if err != nil {
+		return nil, parseErr("virtual files", err)
+	}
+	return res, nil
+}
+
+func (c *connection) CountStagedVirtualFiles(ctx context.Context, projectID, environment string) (int, error) {
+	var res int
+	err := c.getDB(ctx).QueryRowxContext(ctx, `
+		SELECT COUNT(*) FROM virtual_files
+		WHERE project_id=$1 AND environment=$2 AND sync_state != 'synced'
+	`, projectID, environment).Scan(&res)
+	if err != nil {
+		return 0, parseErr("virtual files", err)
+	}
+	return res, nil
+}
+
+func (c *connection) FindProjectIDsForUserFilesAutoSync(ctx context.Context) ([]string, error) {
+	var res []string
+	err := c.getDB(ctx).SelectContext(ctx, &res, `
+		SELECT DISTINCT vf.project_id
+		FROM virtual_files vf
+		JOIN projects p ON p.id = vf.project_id
+		WHERE vf.sync_state != 'synced'
+			AND p.git_remote IS NOT NULL AND p.github_installation_id IS NOT NULL
+			AND p.user_files_sync_interval_seconds > 0
+			AND (p.user_files_synced_on IS NULL OR p.user_files_synced_on + make_interval(secs => p.user_files_sync_interval_seconds) < now())
+	`)
+	if err != nil {
+		return nil, parseErr("virtual files", err)
+	}
+	return res, nil
+}
+
 func (c *connection) UpsertVirtualFile(ctx context.Context, opts *database.InsertVirtualFileOptions) error {
 	if err := database.Validate(opts); err != nil {
 		return err
 	}
 
+	// Any write makes the file dirty again, so a concurrent sync (whose state transitions are gated on
+	// the data it flushed) will not mark a row synced that was edited after the flush.
+	// On the synced-to-dirty transition, the row's last-synced content is snapshotted into base_data:
+	// it is the version this write was based on, and the sync job compares it to the Git copy to detect
+	// edits made directly in Git while the row was dirty.
 	_, err := c.getDB(ctx).ExecContext(ctx, `
-		INSERT INTO virtual_files (project_id, environment, owner_id, path, data, deleted)
-		VALUES ($1, $2, $3, $4, $5, FALSE)
+		INSERT INTO virtual_files (project_id, environment, owner_id, path, data, deleted, sync_state)
+		VALUES ($1, $2, $3, $4, $5, FALSE, 'dirty')
 		ON CONFLICT (project_id, environment, path) DO UPDATE SET
 			data = EXCLUDED.data,
 		    owner_id = EXCLUDED.owner_id,
 			deleted = FALSE,
+			sync_state = 'dirty',
+			base_data = CASE WHEN virtual_files.sync_state = 'synced' THEN virtual_files.data ELSE virtual_files.base_data END,
 			updated_on = now()
 	`, opts.ProjectID, opts.Environment, opts.OwnerID, opts.Path, opts.Data)
 	if err != nil {
@@ -2953,17 +3045,51 @@ func (c *connection) UpsertVirtualFile(ctx context.Context, opts *database.Inser
 }
 
 func (c *connection) UpdateVirtualFileDeleted(ctx context.Context, projectID, environment, path string) error {
+	// A delete is itself a staged change to flush to Git, so it is marked dirty.
+	// Like UpsertVirtualFile, the synced-to-dirty transition snapshots the last-synced content into
+	// base_data (SET expressions read the old row, so base_data sees data before it is cleared).
 	res, err := c.getDB(ctx).ExecContext(ctx, `
 		UPDATE virtual_files SET
 			data = ''::BYTEA,
 			deleted = TRUE,
+			sync_state = 'dirty',
+			base_data = CASE WHEN sync_state = 'synced' THEN data ELSE base_data END,
 			updated_on = now()
 		WHERE project_id=$1 AND environment=$2 AND path=$3`, projectID, environment, path)
 	return checkUpdateRow("virtual file", res, err)
 }
 
+func (c *connection) MarkVirtualFileSynced(ctx context.Context, projectID, environment, path string, expectedData []byte, expectedDeleted bool) (bool, error) {
+	// The update is gated on (data, deleted) so a row edited after the flush stays dirty for the next sync.
+	// updated_on is bumped so the transition reaches runtimes through the FindVirtualFiles feed,
+	// which lets them drop their staged overlay copy and serve the Git copy instead.
+	res, err := c.getDB(ctx).ExecContext(ctx, `
+		UPDATE virtual_files SET
+			sync_state = 'synced',
+			base_data = NULL,
+			updated_on = now()
+		WHERE project_id=$1 AND environment=$2 AND path=$3 AND data=$4 AND deleted=$5 AND sync_state != 'synced'
+	`, projectID, environment, path, expectedData, expectedDeleted)
+	if err != nil {
+		return false, parseErr("virtual file", err)
+	}
+	n, err := res.RowsAffected()
+	if err != nil {
+		return false, parseErr("virtual file", err)
+	}
+	return n > 0, nil
+}
+
 func (c *connection) DeleteExpiredVirtualFiles(ctx context.Context, retention time.Duration) error {
-	_, err := c.getDB(ctx).ExecContext(ctx, `DELETE FROM virtual_files WHERE deleted AND updated_on + $1 < now()`, retention)
+	// Only tombstones may be purged, and for Git-connected projects only after the deletion is confirmed
+	// in Git (synced): purging a dirty tombstone would lose a staged deletion, and purging a live synced
+	// row would lose the path index used to find the file for later edits.
+	_, err := c.getDB(ctx).ExecContext(ctx, `
+		DELETE FROM virtual_files vf
+		USING projects p
+		WHERE p.id = vf.project_id AND vf.deleted AND vf.updated_on + $1 < now()
+			AND (vf.sync_state = 'synced' OR p.git_remote IS NULL OR p.github_installation_id IS NULL)
+	`, retention)
 	return parseErr("virtual files", err)
 }