[{"data":1,"prerenderedAt":613},["ShallowReactive",2],{"doc:\u002Fdocs\u002Fdatabase-migrations":3,"docs-nav":386},{"id":4,"title":5,"body":6,"description":377,"extension":378,"meta":379,"navigation":381,"path":382,"seo":383,"stem":384,"__hash__":385},"docs\u002Fdocs\u002Fdatabase-migrations.md","Database migrations",{"type":7,"value":8,"toc":370},"minimark",[9,13,33,38,116,126,130,199,206,210,252,267,271,278,328,332,366],[10,11,5],"h1",{"id":12},"database-migrations",[14,15,16,17,21,22,32],"p",{},"The gateway keeps its metadata — users, folders, permissions, flows, datasources, runs,\ncontracts — in PostgreSQL. Its schema changes with the code, through ",[18,19,20],"strong",{},"Alembic","\nrevisions in ",[23,24,28],"a",{"href":25,"rel":26},"https:\u002F\u002Fgithub.com\u002FJumpAsAService\u002FTabularia\u002Ftree\u002Fmain\u002Fgateway\u002Fapp\u002Fdb\u002Fmigrations\u002Fversions",[27],"nofollow",[29,30,31],"code",{},"gateway\u002Fapp\u002Fdb\u002Fmigrations\u002Fversions\u002F",".\nThe SQLModel models are the single source of truth: a revision is generated from them,\nread by a person, and committed with the change that needs it.",[34,35,37],"h2",{"id":36},"one-command","One command",[39,40,45],"pre",{"className":41,"code":42,"language":43,"meta":44,"style":44},"language-bash shiki shiki-themes github-dark-dimmed github-light","python -m app.db.migra                  # bring the database to the latest revision\npython -m app.db.migra stato            # the database's revision and the code's\npython -m app.db.migra controlla        # is the database equal to the models? (exit 1 if not)\npython -m app.db.migra nuova \"message\"  # generate a revision from the models\n","bash","",[29,46,47,68,83,98],{"__ignoreMap":44},[48,49,52,56,60,64],"span",{"class":50,"line":51},"line",1,[48,53,55],{"class":54},"sqRhv","python",[48,57,59],{"class":58},"swcJU"," -m",[48,61,63],{"class":62},"sXfbr"," app.db.migra",[48,65,67],{"class":66},"sgHix","                  # bring the database to the latest revision\n",[48,69,71,73,75,77,80],{"class":50,"line":70},2,[48,72,55],{"class":54},[48,74,59],{"class":58},[48,76,63],{"class":62},[48,78,79],{"class":62}," stato",[48,81,82],{"class":66},"            # the database's revision and the code's\n",[48,84,86,88,90,92,95],{"class":50,"line":85},3,[48,87,55],{"class":54},[48,89,59],{"class":58},[48,91,63],{"class":62},[48,93,94],{"class":62}," controlla",[48,96,97],{"class":66},"        # is the database equal to the models? (exit 1 if not)\n",[48,99,101,103,105,107,110,113],{"class":50,"line":100},4,[48,102,55],{"class":54},[48,104,59],{"class":58},[48,106,63],{"class":62},[48,108,109],{"class":62}," nuova",[48,111,112],{"class":62}," \"message\"",[48,114,115],{"class":66},"  # generate a revision from the models\n",[14,117,118,121,122,125],{},[29,119,120],{},"migra"," runs in ",[18,123,124],{},"one transaction"," (PostgreSQL's DDL is transactional: all or nothing),\nunder a Postgres advisory lock, so concurrent runs queue and the second finds the work\ndone. After migrating it compares the database with the models; a difference stops it and\nrolls everything back — two installations of the same version have the same schema.",[34,127,129],{"id":128},"who-runs-it","Who runs it",[131,132,133,146],"table",{},[134,135,136],"thead",{},[137,138,139,143],"tr",{},[140,141,142],"th",{},"Where",[140,144,145],{},"How",[147,148,149,162,173,188],"tbody",{},[137,150,151,155],{},[152,153,154],"td",{},"Development compose",[152,156,157,158,161],{},"the gateway at startup (",[29,159,160],{},"DB__AUTO_MIGRATE=true",")",[137,163,164,167],{},[152,165,166],{},"A compose deployment",[152,168,169,170],{},"a one-shot service running the command; gateways and orchestrators wait for it with ",[29,171,172],{},"depends_on: {migrate: {condition: service_completed_successfully}}",[137,174,175,178],{},[152,176,177],{},"Helm",[152,179,180,181,184,185,161],{},"an init container ",[29,182,183],{},"migrate"," in every gateway and orchestrator pod (",[29,186,187],{},"migrations.enabled",[137,189,190,193],{},[152,191,192],{},"Your own pipeline",[152,194,195,196,161],{},"the same command, before upgrading (and ",[29,197,198],{},"migrations.enabled: false",[14,200,201,202,205],{},"Outside development the application ",[18,203,204],{},"never migrates",": at startup it checks the database\nrevision. Behind its code → it refuses to start and says to run the migration step. Ahead\nof its code → it starts with a warning: that is the previous version during a rolling\nupdate, and revisions are written so that it keeps working (below).",[34,207,209],{"id":208},"adding-a-revision","Adding a revision",[211,212,213,217,224,230,245],"ol",{},[214,215,216],"li",{},"Change the models.",[214,218,219,220,223],{},"On a development database at the latest revision, generate the revision:\n",[29,221,222],{},"docker compose exec --user \"$(id -u):$(id -g)\" -e HOME=\u002Ftmp gateway python -m app.db.migra nuova \"add X to Y\"","\n(as your user: the container's user cannot write into the mounted sources).",[214,225,226,229],{},[18,227,228],{},"Read the file."," Autogenerate does not see everything — a renamed column looks like a\ndrop and an add, a data change is not there at all — and it renders server defaults\nand types the way it understands them. Fix it by hand.",[214,231,232,235,236,239,240,244],{},[29,233,234],{},"python -m app.db.migra",", then ",[29,237,238],{},"python -m app.db.migra controlla"," → ",[241,242,243],"em",{},"equal to the models",".",[214,246,247,248,251],{},"Commit the model change and the revision together. The gateway test\n",[29,249,250],{},"test_migrations.py"," fails if the models and the revisions do not agree.",[14,253,254,255,258,259,262,263,266],{},"Revisions are numbered ",[29,256,257],{},"0002",", ",[29,260,261],{},"0003","… in one chain. They go ",[18,264,265],{},"forward only",": a mistake is\nfixed by a new revision, and a rollback of the database is a restore from backup.",[34,268,270],{"id":269},"rules-for-rolling-updates-expand--contract","Rules for rolling updates (expand \u002F contract)",[14,272,273,274,277],{},"During an upgrade the new revision is applied while pods of the previous version still\nserve requests. A revision must therefore work with ",[18,275,276],{},"the previous version of the code",":",[279,280,281,299,305,315,325],"ul",{},[214,282,283,286,287,290,291,294,295,298],{},[18,284,285],{},"Add"," columns as nullable, or with a server default (",[29,288,289],{},"server_default=",") — the previous\ncode inserts rows without them. Add tables and indexes freely (a large index:\n",[29,292,293],{},"postgresql_concurrently=True"," inside ",[29,296,297],{},"op.get_context().autocommit_block()",").",[214,300,301,304],{},[18,302,303],{},"Never rename or drop in one step."," A rename is three releases: add the new column and\nwrite both; move the readers and copy the data; drop the old one. A drop is two: stop\nusing it; drop it in the next release.",[214,306,307,310,311,314],{},[18,308,309],{},"Tighten constraints in two steps",": fill the column first, make it ",[29,312,313],{},"NOT NULL"," in the\nnext release.",[214,316,317,320,321,324],{},[18,318,319],{},"Data changes"," go in the revision (",[29,322,323],{},"op.execute(...)",") and must be idempotent and quick;\na long backfill belongs to the application, in batches.",[214,326,327],{},"Keep locks short: every statement of a revision runs while the previous version works.",[34,329,331],{"id":330},"before-alembic","Before Alembic",[14,333,334,335,338,339,342,343,346,347,350,351,353,354,361,362,365],{},"Until 2026-10 the gateway created its schema at startup: ",[29,336,337],{},"create_all"," plus a list of\nidempotent ",[29,340,341],{},"ALTER … IF NOT EXISTS",". That left drift between installations — a database\ncreated from scratch and one upgraded over months had different foreign keys, types and\nserver defaults. A database from that time (tables, no ",[29,344,345],{},"alembic_version",") is ",[18,348,349],{},"adopted","\nby the first ",[29,352,120],{},": the frozen list brings it to the last pre-Alembic state\n(",[23,355,358],{"href":356,"rel":357},"https:\u002F\u002Fgithub.com\u002FJumpAsAService\u002FTabularia\u002Fblob\u002Fmain\u002Fgateway\u002Fapp\u002Fdb\u002Flegacy.py",[27],[29,359,360],{},"app\u002Fdb\u002Flegacy.py","), the drift is realigned to the models,\nit is stamped at the baseline ",[29,363,364],{},"0001"," and compared with the models. Nothing to do by hand.",[367,368,369],"style",{},"html pre.shiki code .sqRhv, html code.shiki .sqRhv{--shiki-default:#F69D50;--shiki-light:#6F42C1}html pre.shiki code .swcJU, html code.shiki .swcJU{--shiki-default:#6CB6FF;--shiki-light:#005CC5}html pre.shiki code .sXfbr, html code.shiki .sXfbr{--shiki-default:#96D0FF;--shiki-light:#032F62}html pre.shiki code .sgHix, html code.shiki .sgHix{--shiki-default:#768390;--shiki-light:#6A737D}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}",{"title":44,"searchDepth":70,"depth":85,"links":371},[372,373,374,375,376],{"id":36,"depth":70,"text":37},{"id":128,"depth":70,"text":129},{"id":208,"depth":70,"text":209},{"id":269,"depth":70,"text":270},{"id":330,"depth":70,"text":331},"The gateway keeps its metadata — users, folders, permissions, flows, datasources, runs,\ncontracts — in PostgreSQL. Its schema changes with the code, through Alembic\nrevisions in gateway\u002Fapp\u002Fdb\u002Fmigrations\u002Fversions\u002F.\nThe SQLModel models are the single source of truth: a revision is generated from them,\nread by a person, and committed with the change that needs it.","md",{"source":380},"docs\u002Fdatabase-migrations.md",true,"\u002Fdocs\u002Fdatabase-migrations",{"title":5,"description":377},"docs\u002Fdatabase-migrations","r0HBYo0eK1HmRyKjRri5GcEYtcawpx1JTn4TORAJ_9E",[387,406,462,494,526,555,581,605],{"title":388,"icon":389,"intro":390,"items":391},"Getting started","start","Install it on one machine, learn the few ideas everything else is built on, and run a first flow.",[392,394,397,400,403],{"path":393,"title":388},"\u002Fdocs\u002Fgetting-started",{"path":395,"title":396},"\u002Fdocs\u002Fgetting-started-docker-compose","Walkthrough with Docker Compose",{"path":398,"title":399},"\u002Fdocs\u002Fdeploy\u002Fminikube","Try it on your computer (minikube)",{"path":401,"title":402},"\u002Fdocs\u002Fconcepts","Concepts",{"path":404,"title":405},"\u002Fdocs\u002Foverview","Overview (README)",{"title":407,"icon":408,"intro":409,"items":410},"Capabilities","capabilities","What you can build: visual flows, analysis on any step, AI questions, pixel-perfect reports, dashboards, Python, R and SQL scripts, containers and the orchestration that ties them together.",[411,414,417,420,423,426,429,432,435,438,441,444,447,450,453,456,459],{"path":412,"title":413},"\u002Fdocs\u002Fcapabilities","All capabilities",{"path":415,"title":416},"\u002Fdocs\u002Fcapabilities\u002Fflows","Low-code flows",{"path":418,"title":419},"\u002Fdocs\u002Fcapabilities\u002Fpreview-and-analysis","Preview and analysis",{"path":421,"title":422},"\u002Fdocs\u002Fcapabilities\u002Fask-your-data","Ask your data with AI",{"path":424,"title":425},"\u002Fdocs\u002Fcapabilities\u002Freports","Pixel-perfect reports",{"path":427,"title":428},"\u002Fdocs\u002Fcapabilities\u002Freport-functions","Report functions",{"path":430,"title":431},"\u002Fdocs\u002Fcapabilities\u002Fdashboards","Dashboards",{"path":433,"title":434},"\u002Fdocs\u002Fscripting","Python and R",{"path":436,"title":437},"\u002Fdocs\u002Fsql-scripts","SQL scripts",{"path":439,"title":440},"\u002Fdocs\u002Fcapabilities\u002Fnotebooks","marimo notebooks as apps",{"path":442,"title":443},"\u002Fdocs\u002Fcontainer-node","Docker images in flows",{"path":445,"title":446},"\u002Fdocs\u002Fcapabilities\u002Fflow-groups","Flow groups: orchestration",{"path":448,"title":449},"\u002Fdocs\u002Fcapabilities\u002Fscheduling","Scheduling",{"path":451,"title":452},"\u002Fdocs\u002Fpushmail","Pushmail: datasources, reports and dashboards by email",{"path":454,"title":455},"\u002Fdocs\u002Fplaceholders","Dynamic dates",{"path":457,"title":458},"\u002Fdocs\u002Fcapabilities\u002Fengines","Engines",{"path":460,"title":461},"\u002Fdocs\u002Fengines\u002Fengine-differences","Engine differences",{"title":463,"icon":464,"intro":465,"items":466},"Sources","sources","Every way data comes in: files, databases, buckets, SFTP folders, HTTP APIs, SharePoint, Google Sheets, and the results of your own flows.",[467,470,473,476,479,482,485,488,491],{"path":468,"title":469},"\u002Fdocs\u002Fsources","All sources",{"path":471,"title":472},"\u002Fdocs\u002Fsources\u002Ffiles","Uploaded files",{"path":474,"title":475},"\u002Fdocs\u002Fsources\u002Fdatabases","Database tables and queries",{"path":477,"title":478},"\u002Fdocs\u002Fs3-datasource","S3 buckets",{"path":480,"title":481},"\u002Fdocs\u002Fsftp","SFTP folders",{"path":483,"title":484},"\u002Fdocs\u002Fapi-datasource","HTTP APIs",{"path":486,"title":487},"\u002Fdocs\u002Fsources\u002Fsharepoint","Excel on SharePoint",{"path":489,"title":490},"\u002Fdocs\u002Fsources\u002Fgoogle-sheets","Google Sheets",{"path":492,"title":493},"\u002Fdocs\u002Fsources\u002Fflow-outputs","Results of flows and scripts",{"title":495,"icon":496,"intro":497,"items":498},"Connections","connections","The systems Tabularia reads from and writes to, how each connection is set up, and who may use it.",[499,502,505,508,511,513,514,516,517,520,523],{"path":500,"title":501},"\u002Fdocs\u002Fconnections","All connections",{"path":503,"title":504},"\u002Fdocs\u002Fconnections\u002Fdatabases","Databases",{"path":506,"title":507},"\u002Fdocs\u002Fsqlserver","Microsoft SQL Server driver",{"path":509,"title":510},"\u002Fdocs\u002Fconnections\u002Fs3","S3 and compatible storage",{"path":480,"title":512},"SFTP servers",{"path":483,"title":484},{"path":486,"title":515},"SharePoint",{"path":489,"title":490},{"path":518,"title":519},"\u002Fdocs\u002Fconnections\u002Fsmtp","SMTP email",{"path":521,"title":522},"\u002Fdocs\u002Fcontainer-registries","Container registries",{"path":524,"title":525},"\u002Fdocs\u002Fconnections\u002Fdestinations","Writing data out",{"title":527,"icon":528,"intro":529,"items":530},"Integrations","integrations","How Tabularia fits your stack: AI providers, the REST API and webhooks, dbt, OpenLineage, single sign-on, SIEM and monitoring.",[531,534,537,540,543,546,549,552],{"path":532,"title":533},"\u002Fdocs\u002Fintegrations","All integrations",{"path":535,"title":536},"\u002Fdocs\u002Fintegrations\u002Fai","AI: where it is used",{"path":538,"title":539},"\u002Fdocs\u002Fapi","API, tokens and webhooks",{"path":541,"title":542},"\u002Fdocs\u002Fdbt-export","Export to dbt",{"path":544,"title":545},"\u002Fdocs\u002Fopenlineage","OpenLineage",{"path":547,"title":548},"\u002Fdocs\u002Fsso-keycloak","Single sign-on with Keycloak",{"path":550,"title":551},"\u002Fdocs\u002Fsiem-splunk","Audit log to Splunk",{"path":553,"title":554},"\u002Fdocs\u002Fintegrations\u002Fmonitoring","Monitoring",{"title":556,"icon":557,"intro":558,"items":559},"Security & governance","governance","Who may see and do what, what is recorded, and how data quality and personal data are kept in check.",[560,563,566,569,572,575,578],{"path":561,"title":562},"\u002Fdocs\u002Fgovernance","Overview",{"path":564,"title":565},"\u002Fdocs\u002Fgovernance\u002Fpermissions","Permissions",{"path":567,"title":568},"\u002Fdocs\u002Fgovernance\u002Faudit-log","Audit log",{"path":570,"title":571},"\u002Fdocs\u002Fgovernance\u002Fdata-contracts","Data contracts",{"path":573,"title":574},"\u002Fdocs\u002Fdesign\u002Fsso-group-mapping","SSO and group mapping",{"path":576,"title":577},"\u002Fdocs\u002Fexamples\u002Fkeycloak","Keycloak example on a laptop",{"path":579,"title":580},"\u002Fdocs\u002Fengines\u002Fclickhouse-ai-user","A ClickHouse user for the AI assistant",{"title":582,"icon":583,"intro":584,"items":585},"Install & operate","operate","Run it in production on Kubernetes, configure it, upgrade it and keep it healthy.",[586,589,592,595,598,601,604],{"path":587,"title":588},"\u002Fdocs\u002Fdeploy\u002Fkubernetes","Kubernetes and Helm",{"path":590,"title":591},"\u002Fdocs\u002Fdeploy\u002Fkind","A company-like cluster on one computer (kind)",{"path":593,"title":594},"\u002Fdocs\u002Fdeploy\u002Fupgrading","Upgrading, backups and restore",{"path":596,"title":597},"\u002Fdocs\u002Fdeploy\u002Frelease-checklist","Install and upgrade checklist",{"path":599,"title":600},"\u002Fdocs\u002Fconfiguration","Configuration reference",{"path":602,"title":603},"\u002Fdocs\u002Fengine-settings","Settings from the panel",{"path":382,"title":5},{"title":606,"icon":607,"intro":608,"items":609},"Reference","reference","Release process and the remaining reference pages.",[610],{"path":611,"title":612},"\u002Fdocs\u002Frelease-process","Release process",1793656372345]