[{"data":1,"prerenderedAt":1539},["ShallowReactive",2],{"doc:\u002Fdocs\u002Fopenlineage":3,"docs-nav":1314},{"id":4,"title":5,"body":6,"description":1305,"extension":1306,"meta":1307,"navigation":1309,"path":1310,"seo":1311,"stem":1312,"__hash__":1313},"docs\u002Fdocs\u002Fopenlineage.md","OpenLineage: Tabularia in your data catalog",{"type":7,"value":8,"toc":1293},"minimark",[9,13,30,42,49,54,59,163,195,198,333,340,344,368,604,615,636,691,696,733,743,747,750,780,783,789,793,1042,1051,1055,1058,1097,1117,1120,1149,1191,1195,1212,1218,1224,1239,1243,1250,1254,1289],[10,11,5],"h1",{"id":12},"openlineage-tabularia-in-your-data-catalog",[14,15,16,17,21,22,29],"p",{},"Tabularia derives the lineage of its own workspace for the Lineage page. Optionally it\nalso ",[18,19,20],"em",{},"tells"," that lineage to an external catalog in the\n",[23,24,28],"a",{"href":25,"rel":26},"https:\u002F\u002Fopenlineage.io",[27],"nofollow","OpenLineage"," standard, with the official Python client, so\nthat Marquez, DataHub, Astro, OpenMetadata or any other collector sees what every run\nread and wrote next to the lineage it already receives from other tools.",[14,31,32,33,36,37,41],{},"Nothing changes in Tabularia when it is on: events leave ",[18,34,35],{},"after"," the commit that\ncloses a run, from a background thread, and a collector that is down or slow never\nfails a run. The other integrations are listed in ",[23,38,40],{"href":39},"\u002Fdocs\u002Fintegrations","Integrations",".",[14,43,44],{},[45,46],"img",{"alt":47,"src":48},"The Lineage page of Tabularia centred on the flow Margin by Category and Month, with its five input datasources and its output","\u002Fmedia\u002Fguide\u002Flineage\u002Flineage-margin-flow.png",[14,50,51],{},[18,52,53],{},"The Lineage page, inside Tabularia: what the sample flow «Margin by Category and Month» reads and writes. OpenLineage sends the same facts, run by run, to your catalog.",[55,56,58],"h2",{"id":57},"turn-it-on","Turn it on",[60,61,62,75],"table",{},[63,64,65],"thead",{},[66,67,68,72],"tr",{},[69,70,71],"th",{},"Variable",[69,73,74],{},"Meaning",[76,77,78,94,111,121,131,153],"tbody",{},[66,79,80,87],{},[81,82,83],"td",{},[84,85,86],"code",{},"OPENLINEAGE__URL",[81,88,89,90,93],{},"The collector's base URL, e.g. ",[84,91,92],{},"http:\u002F\u002Fmarquez:5000",". Empty = off.",[66,95,96,101],{},[81,97,98],{},[84,99,100],{},"OPENLINEAGE__ENDPOINT",[81,102,103,104,107,108,41],{},"Path of the lineage endpoint, default ",[84,105,106],{},"api\u002Fv1\u002Flineage"," (Marquez, Astro). DataHub: ",[84,109,110],{},"openapi\u002Fopenlineage\u002Fapi\u002Fv1\u002Flineage",[66,112,113,118],{},[81,114,115],{},[84,116,117],{},"OPENLINEAGE__API_KEY",[81,119,120],{},"Bearer token, if the collector wants one.",[66,122,123,128],{},[81,124,125],{},[84,126,127],{},"OPENLINEAGE__FILE",[81,129,130],{},"Write the same events as JSON Lines to this path, instead of or besides the URL (air-gapped setups, custom pipelines).",[66,132,133,138],{},[81,134,135],{},[84,136,137],{},"OPENLINEAGE__NAMESPACE",[81,139,140,141,144,145,148,149,152],{},"The namespace of this installation's jobs, default ",[84,142,143],{},"tabularia",". One per installation (",[84,146,147],{},"tabularia-prod",", ",[84,150,151],{},"tabularia-staging","…).",[66,154,155,160],{},[81,156,157],{},[84,158,159],{},"OPENLINEAGE__TIMEOUT_SECONDS",[81,161,162],{},"HTTP timeout per attempt, default 10; an event is tried three times at most.",[14,164,165,166,148,169,172,173,176,177,182,183,186,187,190,191,194],{},"The library's own configuration works too: ",[84,167,168],{},"OPENLINEAGE_URL",[84,170,171],{},"OPENLINEAGE_API_KEY",",\nor ",[84,174,175],{},"OPENLINEAGE_CONFIG=\u002Fpath\u002Fopenlineage.yml"," for composite or custom transports —\nsee the\n",[23,178,181],{"href":179,"rel":180},"https:\u002F\u002Fopenlineage.io\u002Fdocs\u002Fclient\u002Fpython",[27],"client documentation",". The Kafka\ntransport needs the ",[84,184,185],{},"confluent-kafka"," package, which the image does not ship. Set the variables\non every gateway and orchestrator process (they are the ones that close runs); the\nHelm chart takes them from ",[84,188,189],{},"openlineage.*"," and ",[84,192,193],{},"secrets.openlineageApiKey",". The\ngateway logs one line at start-up saying where events go.",[14,196,197],{},"Quick local collector for a test:",[199,200,205],"pre",{"className":201,"code":202,"language":203,"meta":204,"style":204},"language-sh shiki shiki-themes github-dark-dimmed github-light","docker run -d --name marquez-db -e POSTGRES_USER=marquez -e POSTGRES_PASSWORD=marquez -e POSTGRES_DB=marquez postgres:16-alpine\ndocker run -d --name marquez --link marquez-db -p 5000:5000 -e POSTGRES_HOST=marquez-db -e POSTGRES_USER=marquez -e POSTGRES_PASSWORD=marquez -e POSTGRES_DB=marquez marquezproject\u002Fmarquez\ndocker run -d --name marquez-web --link marquez -p 3005:3000 -e MARQUEZ_HOST=marquez -e MARQUEZ_PORT=5000 marquezproject\u002Fmarquez-web\n","sh","",[84,206,207,249,294],{"__ignoreMap":204},[208,209,212,216,220,224,227,230,233,236,238,241,243,246],"span",{"class":210,"line":211},"line",1,[208,213,215],{"class":214},"sqRhv","docker",[208,217,219],{"class":218},"sXfbr"," run",[208,221,223],{"class":222},"swcJU"," -d",[208,225,226],{"class":222}," --name",[208,228,229],{"class":218}," marquez-db",[208,231,232],{"class":222}," -e",[208,234,235],{"class":218}," POSTGRES_USER=marquez",[208,237,232],{"class":222},[208,239,240],{"class":218}," POSTGRES_PASSWORD=marquez",[208,242,232],{"class":222},[208,244,245],{"class":218}," POSTGRES_DB=marquez",[208,247,248],{"class":218}," postgres:16-alpine\n",[208,250,252,254,256,258,260,263,266,268,271,274,276,279,281,283,285,287,289,291],{"class":210,"line":251},2,[208,253,215],{"class":214},[208,255,219],{"class":218},[208,257,223],{"class":222},[208,259,226],{"class":222},[208,261,262],{"class":218}," marquez",[208,264,265],{"class":222}," --link",[208,267,229],{"class":218},[208,269,270],{"class":222}," -p",[208,272,273],{"class":218}," 5000:5000",[208,275,232],{"class":222},[208,277,278],{"class":218}," POSTGRES_HOST=marquez-db",[208,280,232],{"class":222},[208,282,235],{"class":218},[208,284,232],{"class":222},[208,286,240],{"class":218},[208,288,232],{"class":222},[208,290,245],{"class":218},[208,292,293],{"class":218}," marquezproject\u002Fmarquez\n",[208,295,297,299,301,303,305,308,310,312,314,317,319,322,324,327,330],{"class":210,"line":296},3,[208,298,215],{"class":214},[208,300,219],{"class":218},[208,302,223],{"class":222},[208,304,226],{"class":222},[208,306,307],{"class":218}," marquez-web",[208,309,265],{"class":222},[208,311,262],{"class":218},[208,313,270],{"class":222},[208,315,316],{"class":218}," 3005:3000",[208,318,232],{"class":222},[208,320,321],{"class":218}," MARQUEZ_HOST=marquez",[208,323,232],{"class":222},[208,325,326],{"class":218}," MARQUEZ_PORT=",[208,328,329],{"class":222},"5000",[208,331,332],{"class":218}," marquezproject\u002Fmarquez-web\n",[14,334,335,336,339],{},"then ",[84,337,338],{},"OPENLINEAGE__URL=http:\u002F\u002Fmarquez:5000"," on a network the gateway shares with it.",[55,341,343],{"id":342},"what-is-reported","What is reported",[14,345,346,347,350,351,354,355,358,359,362,363,367],{},"Everything that runs is traced as a flow is: flows, refreshes, SQL scripts, saved\nscripts, flow groups with their cells, the Script and Container nodes of a flow, and\nthe sessions of a notebook. Every run that reaches a terminal state sends a ",[84,348,349],{},"START","\nand a ",[84,352,353],{},"COMPLETE"," (or ",[84,356,357],{},"FAIL",") ",[84,360,361],{},"RunEvent",", with the real start and end times, so the\ncatalog computes durations. A notebook's session is told while it lives (see\n",[23,364,366],{"href":365},"#notebooks-a-session-is-a-run","Notebooks",").",[60,369,370,386],{},[63,371,372],{},[66,373,374,377,380,383],{},[69,375,376],{},"Tabularia",[69,378,379],{},"OpenLineage job",[69,381,382],{},"Inputs",[69,384,385],{},"Outputs",[76,387,388,404,442,458,474,495,518,537,556,572,588],{},[66,389,390,393,398,401],{},[81,391,392],{},"A flow run (manual, scheduled, or from the editor)",[81,394,395],{},[84,396,397],{},"\u003Cfolder path>\u002F\u003Cflow name>",[81,399,400],{},"every datasource the flow's source nodes read; a file uploaded in the editor",[81,402,403],{},"the union of what its output nodes wrote",[66,405,406,409,436,439],{},[81,407,408],{},"An output node of that run",[81,410,411,414,415,418,419,148,422,148,425,428,429,432,433],{},[84,412,413],{},"\u003Cfolder path>\u002F\u003Cflow name>.\u003Coutput>",", child of the flow run (",[84,416,417],{},"ParentRunFacet",") — ",[84,420,421],{},"datasource \u003Cname>",[84,423,424],{},"table \u003Cname>",[84,426,427],{},"file \u003Ckey>"," (for SFTP ",[84,430,431],{},"file \u002F\u003Cfolder>\u002F\u003Cname template>","), ",[84,434,435],{},"email",[81,437,438],{},"the flow's inputs",[81,440,441],{},"the datasource published, the table written, the S3\u002FGCS object, the SFTP file, the mirror copy",[66,443,444,451,454,456],{},[81,445,446,447,450],{},"A nested flow (",[84,448,449],{},"Run flow"," node)",[81,452,453],{},"its outputs stay under the nested flow's own job name",[81,455],{},[81,457],{},[66,459,460,463,468,471],{},[81,461,462],{},"A datasource refresh (database, SharePoint, SFTP, S3 folder or HTTP API), by hand, on schedule or inside a flow",[81,464,465],{},[84,466,467],{},"\u003Cfolder path>\u002F\u003Cdatasource> (refresh)",[81,469,470],{},"the table; the tables a SQL query reads; the SharePoint file; the SFTP or S3 folder and glob; the API's path",[81,472,473],{},"the datasource",[66,475,476,479,489,492],{},[81,477,478],{},"A Script or Container node of a flow",[81,480,481,484,485,488],{},[84,482,483],{},"\u003Cfolder path>\u002F\u003Cflow name>.script \u003Clabel>"," \u002F ",[84,486,487],{},".container \u003Clabel>",", child of the flow run",[81,490,491],{},"the datasources whose data reaches the node's input, and the inputs a saved script it runs declares",[81,493,494],{},"— (what it hands to the next nodes is not a dataset: the Outputs write)",[66,496,497,504,509,516],{},[81,498,499,500,503],{},"A SQL script writing its datasource (",[18,501,502],{},"Refresh now",", its schedule, a flow group's cell)",[81,505,506],{},[84,507,508],{},"\u003Cfolder path>\u002F\u003Cscript name> (SQL script)",[81,510,511,512,515],{},"on the catalog, the datasources the SQL names (as resolved when the write started); on a connection, the tables the SQL reads (found with sqlglot, as for a SQL datasource's refresh; a query it cannot parse gives none, its text stays in the ",[84,513,514],{},"sql"," facet)",[81,517,473],{},[66,519,520,526,531,534],{},[81,521,522,523,503],{},"A saved script's run (",[18,524,525],{},"Run",[81,527,528],{},[84,529,530],{},"\u003Cfolder path>\u002F\u003Cscript name> (script)",[81,532,533],{},"the datasources it declares as inputs",[81,535,536],{},"the datasource it publishes, if it does",[66,538,539,542,547,553],{},[81,540,541],{},"A notebook's session (the app, or the editor)",[81,543,544],{},[84,545,546],{},"\u003Cfolder path>\u002F\u003Cnotebook name> (notebook)",[81,548,549,550],{},"the datasources it reads with ",[84,551,552],{},"tabularia.read",[81,554,555],{},"— (notebooks only read)",[66,557,558,561,566,569],{},[81,559,560],{},"A flow group's run",[81,562,563],{},[84,564,565],{},"\u003Cfolder path>\u002F\u003Cgroup name> (flow group)",[81,567,568],{},"the union of what its cells read",[81,570,571],{},"the union of what its cells wrote",[66,573,574,577,583,586],{},[81,575,576],{},"A Container cell of a flow group",[81,578,579,582],{},[84,580,581],{},"\u003Cgroup's job>.container \u003Clabel>",", child of the group run",[81,584,585],{},"—",[81,587,585],{},[66,589,590,593,598,601],{},[81,591,592],{},"A Pushmail cell of a flow group",[81,594,595,582],{},[84,596,597],{},"\u003Cgroup's job>.pushmail \u003Clabel>",[81,599,600],{},"the datasources it attaches",[81,602,603],{},"the email (see below)",[14,605,606,607,610,611,614],{},"The kind in brackets keeps a SQL script, a saved script, a notebook and a flow group\napart from a flow with the same name in the same folder. An Output node's email is a\ndataset like a Pushmail cell's: namespace ",[84,608,609],{},"smtp:\u002F\u002F\u003Chost>:\u003Cport>",", name\n",[84,612,613],{},"\u002F\u003Cfolder path>\u002F\u003Cflow name>\u002Femail"," (one dataset for a flow's emails through one server),\nnever the recipients.",[14,616,617,621,622,624,625,628,629,631,632,635],{},[618,619,620],"strong",{},"One job per node, whatever started it."," Marquez nests a job under its parent's\nname when a run carries a ",[84,623,417],{}," (",[84,626,627],{},"\u003Cparent>.\u003Cchild>","). So the parent facet\nis sent only for the steps that exist only inside their parent: the Output, Script\nand Container nodes of the flow itself, and the Container and Pushmail cells of a\nflow group, which is where that hierarchy is wanted. Everything with a life of its\nown — a refresh triggered by a flow, the outputs of a nested flow, and the flows,\nrefreshes, SQL scripts and saved scripts a flow group runs — keeps the same job it\nhas when it runs on its own, and points at the orchestration or the group run that\ncontained it through the ",[84,630,143],{}," run facet (",[84,633,634],{},"parentRunId","). The same object run\nfrom the editor, on its schedule or inside a group always lands in the same job.",[14,637,638,639,641,642,647,648,624,651,654,655,148,658,624,661,664,665,432,668,148,671,673,674,677,678,680,681,624,684,687,688,367],{},"Every run carries a ",[84,640,143],{}," run facet\n(",[23,643,646],{"href":644,"rel":645},"https:\u002F\u002Fgithub.com\u002FJumpAsAService\u002FTabularia\u002Fblob\u002Fmain\u002Fdocs\u002Fopenlineage\u002FTabulariaRunFacet.json",[27],"schema","): ",[84,649,650],{},"runId",[84,652,653],{},"GET \u002Fruns\u002F{runId}","),\n",[84,656,657],{},"kind",[84,659,660],{},"trigger",[84,662,663],{},"manual","\u002F",[84,666,667],{},"schedule",[84,669,670],{},"engine",[84,672,634],{},", and ",[84,675,676],{},"version"," (the\nSQL script's, saved script's or notebook's version that ran); a notebook's session\nhas no ",[84,679,650],{}," but its ",[84,682,683],{},"mode",[84,685,686],{},"app"," or ",[84,689,690],{},"edit",[692,693,695],"h3",{"id":694},"flow-groups","Flow groups",[14,697,698,699,701,702,705,706,647,710,148,713,716,717,720,721,724,725,728,729,732],{},"A group run is the parent of its cells, as a flow's run is of its Outputs. Its job\ncarries what its cells read and wrote; the Container and Pushmail cells are its child\njobs; the flows, refreshes, SQL scripts and saved scripts it runs keep their own jobs\nand name the group run in ",[84,700,634],{},". A Container cell's job, and a Container\nnode's, carries the image in the ",[84,703,704],{},"tabularia_container"," job facet\n(",[23,707,646],{"href":708,"rel":709},"https:\u002F\u002Fgithub.com\u002FJumpAsAService\u002FTabularia\u002Fblob\u002Fmain\u002Fdocs\u002Fopenlineage\u002FTabulariaContainerJobFacet.json",[27],[84,711,712],{},"registry",[84,714,715],{},"registryHost",",\n",[84,718,719],{},"repository",", the ",[84,722,723],{},"tag"," asked for and the ",[84,726,727],{},"digest"," it was resolved to when the run\nstarted, and the full ",[84,730,731],{},"image"," reference.",[14,734,735,736,738,739,742],{},"The email of a Pushmail cell is a dataset: namespace ",[84,737,609],{}," (the SMTP\nconnection's server, port 587 when the connection does not set one), name\n",[84,740,741],{},"\u002F\u003Cfolder path>\u002F\u003Cgroup name>\u002F\u003Ccell label>",". The recipients are not in it, nor anywhere\nin the events: e-mail addresses do not leave the installation.",[692,744,746],{"id":745},"notebooks-a-session-is-a-run","Notebooks: a session is a run",[14,748,749],{},"The job is the notebook; a run is a session of its pod, from the moment it starts to\nthe moment it stops (by hand, idle, or replaced by a new version). Tabularia sends:",[751,752,753,759,768],"ul",{},[754,755,756,758],"li",{},[84,757,349],{}," when the pod is created, with no inputs (nothing has been read yet);",[754,760,761,764,765,767],{},[84,762,763],{},"RUNNING"," the first time the session reads a datasource with ",[84,766,552],{},", with\nthat datasource as its input — once per datasource and session, however many times\nthe cells run again;",[754,769,770,772,773,775,776,779],{},[84,771,353],{}," when the session stops (or the notebook is deleted while it runs), with\nevery datasource it read; ",[84,774,357],{}," when the pod could not start, with the reason;\n",[84,777,778],{},"ABORT"," when it was stopped before it started.",[14,781,782],{},"The app and the editor of a notebook can run at the same time: each session has its own\nreads, so one reading a datasource does not take it from the other's closing event.",[14,784,785,786,788],{},"This follows the standard: a run's datasets accumulate over its events, and ",[84,787,763],{},"\nis the event meant for what a run learns while it runs. The closing event repeats\nevery input so that collectors which only look at the terminal event still see them.\nA notebook only reads: its runs have no outputs. The reads are also kept for the\nLineage page, which draws a notebook as a reader of every datasource it has read.",[692,790,792],{"id":791},"dataset-names","Dataset names",[60,794,795,815],{},[63,796,797],{},[66,798,799,802,807,812],{},[69,800,801],{},"Dataset",[69,803,804],{},[84,805,806],{},"namespace",[69,808,809],{},[84,810,811],{},"name",[69,813,814],{},"Facets",[76,816,817,863,925,957,976,995,1024],{},[66,818,819,822,827,833],{},[81,820,821],{},"A Tabularia datasource",[81,823,824],{},[84,825,826],{},"tabularia:\u002F\u002F\u003Cnamespace>",[81,828,829,832],{},[84,830,831],{},"\u002F\u003Cfolder path>\u002F\u003Cname>"," — the name you see in Tabularia",[81,834,835,148,837,148,840,843,844,847,848,851,852,855,856,859,860,862],{},[84,836,646],{},[84,838,839],{},"documentation",[84,841,842],{},"datasetVersion"," (the snapshot key), ",[84,845,846],{},"symlinks"," → its folder in the bucket (",[84,849,850],{},"s3:\u002F\u002Fbucket"," + ",[84,853,854],{},"\u002Fdatasets\u002F\u003Cid>",", stable across snapshots), ",[84,857,858],{},"dataSource"," (uri of the parquet it serves now), ",[84,861,143],{}," (see below)",[66,864,865,868,888,906],{},[81,866,867],{},"A database table",[81,869,870,148,873,148,876,148,879,148,882,148,885],{},[84,871,872],{},"postgres:\u002F\u002Fhost:5432",[84,874,875],{},"mysql:\u002F\u002Fhost:3306",[84,877,878],{},"clickhouse:\u002F\u002Fhost:8443",[84,880,881],{},"trino:\u002F\u002Fhost:8080",[84,883,884],{},"oracle:\u002F\u002Fhost:1521",[84,886,887],{},"mssql:\u002F\u002Fhost:1433",[81,889,890,893,894,897,898,901,902,905],{},[84,891,892],{},"db.schema.table"," (Postgres, Trino, SQL Server; ",[84,895,896],{},"dbo"," when no schema is given) · ",[84,899,900],{},"db.table"," (MySQL, ClickHouse) · ",[84,903,904],{},"service.SCHEMA.TABLE"," (Oracle, with Oracle's upper case for unquoted names)",[81,907,908,910,911,624,914,917,918,921,922],{},[84,909,858],{},"; on outputs ",[84,912,913],{},"lifecycleStateChange",[84,915,916],{},"OVERWRITE"," for replace, ",[84,919,920],{},"ALTER"," for append) and ",[84,923,924],{},"outputStatistics",[66,926,927,930,937,947],{},[81,928,929],{},"An object in a bucket",[81,931,932,687,934],{},[84,933,850],{},[84,935,936],{},"gs:\u002F\u002Fbucket",[81,938,939,942,943,946],{},[84,940,941],{},"\u002Fpath\u002Fkey"," — for an «S3 folder» datasource ",[84,944,945],{},"\u002Ffolder\u002Fglob",", for an Output key with placeholders the template (one dataset per Output)",[81,948,949,148,952,954,955],{},[84,950,951],{},"storage",[84,953,842],{}," (the key a run wrote, when the key has placeholders), ",[84,956,924],{},[66,958,959,962,967,972],{},[81,960,961],{},"A SharePoint file",[81,963,964],{},[84,965,966],{},"sharepoint:\u002F\u002Fsite host\u002Fsites\u002F…",[81,968,969],{},[84,970,971],{},"\u002Fpath\u002Ffile.xlsx#Sheet",[81,973,974],{},[84,975,858],{},[66,977,978,981,987,993],{},[81,979,980],{},"An HTTP API call",[81,982,983,986],{},[84,984,985],{},"https:\u002F\u002Fhost[:port]"," (the base URL's origin)",[81,988,989,990],{},"the base path + the call's path as written, placeholders included, never the query string: ",[84,991,992],{},"\u002Fv1\u002F{today-30}..",[81,994,585],{},[66,996,997,1000,1005,1015],{},[81,998,999],{},"Files on an SFTP server",[81,1001,1002],{},[84,1003,1004],{},"sftp:\u002F\u002Fhost:port",[81,1006,1007,1008,1010,1011,1014],{},"input: ",[84,1009,945],{},"; output: ",[84,1012,1013],{},"\u002Ffolder\u002F\u003Cfile-name template>",", one dataset per Output",[81,1016,1017,148,1019,1021,1022],{},[84,1018,951],{},[84,1020,842],{}," (the file the run wrote), ",[84,1023,924],{},[66,1025,1026,1029,1033,1038],{},[81,1027,1028],{},"A file uploaded in the editor",[81,1030,1031],{},[84,1032,850],{},[81,1034,1035],{},[84,1036,1037],{},"\u002Fuploads\u002F…parquet",[81,1039,1040],{},[84,1041,951],{},[14,1043,1044,1045,1050],{},"Tables and objects follow the\n",[23,1046,1049],{"href":1047,"rel":1048},"https:\u002F\u002Fopenlineage.io\u002Fdocs\u002Fspec\u002Fnaming",[27],"OpenLineage naming conventions",", so a\ncatalog that also receives lineage from dbt, Airflow or Spark matches the same\nnodes.",[692,1052,1054],{"id":1053},"renames-and-moves","Renames and moves",[14,1056,1057],{},"The folder path is part of the names of flows, datasources, SQL scripts, saved\nscripts, notebooks and flow groups, so renaming or moving one of them, or a folder,\nrenames them in the catalog too. Right after the change Tabularia sends:",[751,1059,1060,1078,1085,1091],{},[754,1061,1062,1063,1066,1067,1069,1070,1073,1074,1077],{},"for a datasource, a ",[84,1064,1065],{},"DatasetEvent"," under the new name carrying the standard\n",[84,1068,913],{}," facet ",[84,1071,1072],{},"RENAME",", whose ",[84,1075,1076],{},"previousIdentifier"," is the old name,\nand the static job of its refresh (database datasources);",[754,1079,1080,1081,1084],{},"for a flow, its static ",[84,1082,1083],{},"JobEvent"," under the new name; its Output, Script and\nContainer jobs come back with its next run;",[754,1086,1087,1088,1090],{},"for a SQL script, a saved script, a notebook or a flow group, its static ",[84,1089,1083],{},"\nunder the new name (with what it reads and writes; a group's cells come back with\nits next run);",[754,1092,1093,1094,1096],{},"for every flow that reads or writes a renamed datasource, and every saved script\nand notebook that reads it, its ",[84,1095,1083],{}," again, so the graph points at the new\nname.",[14,1098,1099,1100,1103,1104,1106,1107,1110,1111,1113,1114,1116],{},"On ",[618,1101,1102],{},"Marquez"," (the collector's endpoint is ",[84,1105,106],{},") the old names are then\ndeleted through its API (",[84,1108,1109],{},"DELETE \u002Fapi\u002Fv1\u002Fnamespaces\u002F{namespace}\u002Fjobs|datasets\u002F{name}",",\na soft delete): the old dataset, the old job, the old Output, Script, Container and\nrefresh jobs, a group's old cell jobs and the emails of its Pushmail cells, the email\nof a flow's email Outputs. Other\ncollectors receive the ",[84,1112,1072],{}," facet and drop the old name their own way. The id in\nthe ",[84,1115,143],{}," facet ties the old and the new dataset in any case.",[14,1118,1119],{},"An Output node finds its datasource by name: after renaming a datasource, update the\nOutput nodes that write it, or their next run creates a datasource with the old name.",[14,1121,1122,1123,1125,1126,1133,1134,148,1137,148,1140,190,1142,1145,1146,41],{},"The ",[84,1124,143],{}," facet on datasources — schema in\n",[23,1127,1130],{"href":1128,"rel":1129},"https:\u002F\u002Fgithub.com\u002FJumpAsAService\u002FTabularia\u002Fblob\u002Fmain\u002Fdocs\u002Fopenlineage\u002FTabulariaDatasetFacet.json",[27],[84,1131,1132],{},"docs\u002Fopenlineage\u002FTabulariaDatasetFacet.json","\n— carries ",[84,1135,1136],{},"datasourceId",[84,1138,1139],{},"projectId",[84,1141,657],{},[84,1143,1144],{},"folder",", so a catalog can get\nback to the object: ",[84,1147,1148],{},"GET \u002Fdatasources\u002F{datasourceId}",[14,1150,1151,1152,624,1155,148,1158,148,1161,148,1164,148,1167,148,1170,716,1173,148,1176,148,1179,432,1182,1184,1185,1187,1188,41],{},"Jobs carry ",[84,1153,1154],{},"jobType",[84,1156,1157],{},"FLOW",[84,1159,1160],{},"OUTPUT",[84,1162,1163],{},"REFRESH",[84,1165,1166],{},"SCRIPT",[84,1168,1169],{},"CONTAINER",[84,1171,1172],{},"SQL_SCRIPT",[84,1174,1175],{},"NOTEBOOK",[84,1177,1178],{},"FLOW_GROUP",[84,1180,1181],{},"PUSHMAIL",[84,1183,839],{}," (the object's description)\nand, for a SQL-query datasource and a SQL script, ",[84,1186,514],{}," (the query, with its dialect).\nFailed runs carry ",[84,1189,1190],{},"errorMessage",[55,1192,1194],{"id":1193},"static-export-dependencies-without-running","Static export: dependencies without running",[14,1196,1197,1200,1201,1203,1204,1207,1208,1211],{},[84,1198,1199],{},"GET \u002Fflows\u002F{id}\u002Fopenlineage"," returns a ",[84,1202,1083],{}," built from the flow's\n",[18,1205,1206],{},"definition",": the job, what it reads, what it writes — without executing anything.\nIt is also in the Flows page, under ",[18,1209,1210],{},"Export",". A datasource the flow will create at\nits first run already has the name it will have.",[14,1213,1214,1217],{},[84,1215,1216],{},"POST \u002Fflows\u002F{id}\u002Fopenlineage"," sends that event to the configured collector: use it\nto populate a catalog with the flows that exist before any of them runs, or after\nchanging a flow's definition. Both need VIEW on the flow's folder.",[14,1219,1220],{},[45,1221],{"alt":1222,"src":1223},"The Export flow dialog of Top 20 Products by Revenue with two choices: dbt and OpenLineage","\u002Fmedia\u002Fguide\u002Fdbt\u002Fexport-flow-menu.png",[14,1225,1226],{},[18,1227,1228,1231,1232,1234,1235,1238],{},[618,1229,1230],{},"Flows"," → a flow's export button: ",[618,1233,28],{}," downloads the static event, ",[618,1236,1237],{},"dbt"," the project.",[55,1240,1242],{"id":1241},"what-leaves-the-installation","What leaves the installation",[14,1244,1245,1246,1249],{},"Only metadata: folder and object names, descriptions, datasource names and column\nnames with their types and descriptions, table names, the text of the SQL query of a\nquery-based datasource and of a SQL script, bucket paths, image references, SMTP\nservers, row counts, run times and the error message of a failed run, with any e-mail\naddress in it replaced by ",[84,1247,1248],{},"\u003Caddress>"," (an error can name whose rights a run had, or a\nrecipient an SMTP server refused). No rows of data, no user names or e-mail addresses\n(not even a Pushmail's recipients), no credentials (connection passwords never reach\nthe gateway's events). Mind that a query text or an error message can quote a literal\nvalue: if that matters, keep the collector inside the same trust boundary as Tabularia.",[55,1251,1253],{"id":1252},"guarantees-and-limits","Guarantees and limits",[751,1255,1256,1265,1271,1277,1283],{},[754,1257,1258,1261,1262,1264],{},[618,1259,1260],{},"Never in the way."," Events are built in the transaction that closes the run,\nfrom the data already written, and handed to a worker thread through a bounded\nqueue (500 closed runs per process, at most ~30 MB): closing a run never waits for\nthe collector. The\nthread tries each event three times at most, ",[84,1263,159],{}," each;\nif the collector refuses or does not answer, the run is still closed and a warning\nwith the run's id is in the log of the process that closed it. When the queue is full — a collector down for long — new\nevents are dropped and the log says so once a minute. A process that stops (a\nrolling update) waits up to 5 seconds for its queue to drain, no more. An event is\nnot retried later: lineage is a chronicle, not a contract; the static export can\nrebuild the dependencies at any time.",[754,1266,1267,1270],{},[618,1268,1269],{},"Cost."," Building a run's events in the closing transaction takes 5–15 ms of\nPython and 6–17 reads (a datasource of 150 columns, a flow with three sources);\nserialising an event takes 3 ms in the worker thread; a full queue of 500 wide runs\nholds about 28 MB. Under a storm of runs (eleven flows in a loop, about two runs\nclosing per second) the API processes serve the same requests per second with\nlineage on or off, whether the collector is fast or answers in 2 seconds, and the\nprocesses that emit spend about 5 points of a core more. A collector running on\nthe same machine is a different matter: Marquez takes about one core while it\ningests, and that is what slows everything else down.",[754,1272,1273,1276],{},[618,1274,1275],{},"Replicas."," Each gateway and orchestrator process sends its own events; the run\nid is a UUID derived from the run's id in Tabularia, the same from any process.",[754,1278,1279,1282],{},[618,1280,1281],{},"Dataset level."," Column-level lineage is not emitted yet.",[754,1284,1285,1288],{},[84,1286,1287],{},"Foreach"," loops report the datasets of the flow, not each iteration.",[1290,1291,1292],"style",{},"html pre.shiki code .sqRhv, html code.shiki .sqRhv{--shiki-default:#F69D50;--shiki-light:#6F42C1}html pre.shiki code .sXfbr, html code.shiki .sXfbr{--shiki-default:#96D0FF;--shiki-light:#032F62}html pre.shiki code .swcJU, html code.shiki .swcJU{--shiki-default:#6CB6FF;--shiki-light:#005CC5}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}",{"title":204,"searchDepth":251,"depth":296,"links":1294},[1295,1296,1302,1303,1304],{"id":57,"depth":251,"text":58},{"id":342,"depth":251,"text":343,"children":1297},[1298,1299,1300,1301],{"id":694,"depth":296,"text":695},{"id":745,"depth":296,"text":746},{"id":791,"depth":296,"text":792},{"id":1053,"depth":296,"text":1054},{"id":1193,"depth":251,"text":1194},{"id":1241,"depth":251,"text":1242},{"id":1252,"depth":251,"text":1253},"Tabularia derives the lineage of its own workspace for the Lineage page. Optionally it\nalso tells that lineage to an external catalog in the\nOpenLineage standard, with the official Python client, so\nthat Marquez, DataHub, Astro, OpenMetadata or any other collector sees what every run\nread and wrote next to the lineage it already receives from other tools.","md",{"source":1308},"docs\u002Fopenlineage.md",true,"\u002Fdocs\u002Fopenlineage",{"title":5,"description":1305},"docs\u002Fopenlineage","f3i9i6IOa3ZjB8M87PvE0xgFNG9f32lPKtDUDOLASSw",[1315,1334,1390,1422,1454,1479,1505,1531],{"title":1316,"icon":1317,"intro":1318,"items":1319},"Getting started","start","Install it on one machine, learn the few ideas everything else is built on, and run a first flow.",[1320,1322,1325,1328,1331],{"path":1321,"title":1316},"\u002Fdocs\u002Fgetting-started",{"path":1323,"title":1324},"\u002Fdocs\u002Fgetting-started-docker-compose","Walkthrough with Docker Compose",{"path":1326,"title":1327},"\u002Fdocs\u002Fdeploy\u002Fminikube","Try it on your computer (minikube)",{"path":1329,"title":1330},"\u002Fdocs\u002Fconcepts","Concepts",{"path":1332,"title":1333},"\u002Fdocs\u002Foverview","Overview (README)",{"title":1335,"icon":1336,"intro":1337,"items":1338},"Capabilities","capabilities","What you can build: visual flows, analysis on any step, AI questions, pixel-perfect reports, dashboards, Python, R and SQL scripts, containers and the orchestration that ties them together.",[1339,1342,1345,1348,1351,1354,1357,1360,1363,1366,1369,1372,1375,1378,1381,1384,1387],{"path":1340,"title":1341},"\u002Fdocs\u002Fcapabilities","All capabilities",{"path":1343,"title":1344},"\u002Fdocs\u002Fcapabilities\u002Fflows","Low-code flows",{"path":1346,"title":1347},"\u002Fdocs\u002Fcapabilities\u002Fpreview-and-analysis","Preview and analysis",{"path":1349,"title":1350},"\u002Fdocs\u002Fcapabilities\u002Fask-your-data","Ask your data with AI",{"path":1352,"title":1353},"\u002Fdocs\u002Fcapabilities\u002Freports","Pixel-perfect reports",{"path":1355,"title":1356},"\u002Fdocs\u002Fcapabilities\u002Freport-functions","Report functions",{"path":1358,"title":1359},"\u002Fdocs\u002Fcapabilities\u002Fdashboards","Dashboards",{"path":1361,"title":1362},"\u002Fdocs\u002Fscripting","Python and R",{"path":1364,"title":1365},"\u002Fdocs\u002Fsql-scripts","SQL scripts",{"path":1367,"title":1368},"\u002Fdocs\u002Fcapabilities\u002Fnotebooks","marimo notebooks as apps",{"path":1370,"title":1371},"\u002Fdocs\u002Fcontainer-node","Docker images in flows",{"path":1373,"title":1374},"\u002Fdocs\u002Fcapabilities\u002Fflow-groups","Flow groups: orchestration",{"path":1376,"title":1377},"\u002Fdocs\u002Fcapabilities\u002Fscheduling","Scheduling",{"path":1379,"title":1380},"\u002Fdocs\u002Fpushmail","Pushmail: datasources, reports and dashboards by email",{"path":1382,"title":1383},"\u002Fdocs\u002Fplaceholders","Dynamic dates",{"path":1385,"title":1386},"\u002Fdocs\u002Fcapabilities\u002Fengines","Engines",{"path":1388,"title":1389},"\u002Fdocs\u002Fengines\u002Fengine-differences","Engine differences",{"title":1391,"icon":1392,"intro":1393,"items":1394},"Sources","sources","Every way data comes in: files, databases, buckets, SFTP folders, HTTP APIs, SharePoint, Google Sheets, and the results of your own flows.",[1395,1398,1401,1404,1407,1410,1413,1416,1419],{"path":1396,"title":1397},"\u002Fdocs\u002Fsources","All sources",{"path":1399,"title":1400},"\u002Fdocs\u002Fsources\u002Ffiles","Uploaded files",{"path":1402,"title":1403},"\u002Fdocs\u002Fsources\u002Fdatabases","Database tables and queries",{"path":1405,"title":1406},"\u002Fdocs\u002Fs3-datasource","S3 buckets",{"path":1408,"title":1409},"\u002Fdocs\u002Fsftp","SFTP folders",{"path":1411,"title":1412},"\u002Fdocs\u002Fapi-datasource","HTTP APIs",{"path":1414,"title":1415},"\u002Fdocs\u002Fsources\u002Fsharepoint","Excel on SharePoint",{"path":1417,"title":1418},"\u002Fdocs\u002Fsources\u002Fgoogle-sheets","Google Sheets",{"path":1420,"title":1421},"\u002Fdocs\u002Fsources\u002Fflow-outputs","Results of flows and scripts",{"title":1423,"icon":1424,"intro":1425,"items":1426},"Connections","connections","The systems Tabularia reads from and writes to, how each connection is set up, and who may use it.",[1427,1430,1433,1436,1439,1441,1442,1444,1445,1448,1451],{"path":1428,"title":1429},"\u002Fdocs\u002Fconnections","All connections",{"path":1431,"title":1432},"\u002Fdocs\u002Fconnections\u002Fdatabases","Databases",{"path":1434,"title":1435},"\u002Fdocs\u002Fsqlserver","Microsoft SQL Server driver",{"path":1437,"title":1438},"\u002Fdocs\u002Fconnections\u002Fs3","S3 and compatible storage",{"path":1408,"title":1440},"SFTP servers",{"path":1411,"title":1412},{"path":1414,"title":1443},"SharePoint",{"path":1417,"title":1418},{"path":1446,"title":1447},"\u002Fdocs\u002Fconnections\u002Fsmtp","SMTP email",{"path":1449,"title":1450},"\u002Fdocs\u002Fcontainer-registries","Container registries",{"path":1452,"title":1453},"\u002Fdocs\u002Fconnections\u002Fdestinations","Writing data out",{"title":40,"icon":1455,"intro":1456,"items":1457},"integrations","How Tabularia fits your stack: AI providers, the REST API and webhooks, dbt, OpenLineage, single sign-on, SIEM and monitoring.",[1458,1460,1463,1466,1469,1470,1473,1476],{"path":39,"title":1459},"All integrations",{"path":1461,"title":1462},"\u002Fdocs\u002Fintegrations\u002Fai","AI: where it is used",{"path":1464,"title":1465},"\u002Fdocs\u002Fapi","API, tokens and webhooks",{"path":1467,"title":1468},"\u002Fdocs\u002Fdbt-export","Export to dbt",{"path":1310,"title":28},{"path":1471,"title":1472},"\u002Fdocs\u002Fsso-keycloak","Single sign-on with Keycloak",{"path":1474,"title":1475},"\u002Fdocs\u002Fsiem-splunk","Audit log to Splunk",{"path":1477,"title":1478},"\u002Fdocs\u002Fintegrations\u002Fmonitoring","Monitoring",{"title":1480,"icon":1481,"intro":1482,"items":1483},"Security & governance","governance","Who may see and do what, what is recorded, and how data quality and personal data are kept in check.",[1484,1487,1490,1493,1496,1499,1502],{"path":1485,"title":1486},"\u002Fdocs\u002Fgovernance","Overview",{"path":1488,"title":1489},"\u002Fdocs\u002Fgovernance\u002Fpermissions","Permissions",{"path":1491,"title":1492},"\u002Fdocs\u002Fgovernance\u002Faudit-log","Audit log",{"path":1494,"title":1495},"\u002Fdocs\u002Fgovernance\u002Fdata-contracts","Data contracts",{"path":1497,"title":1498},"\u002Fdocs\u002Fdesign\u002Fsso-group-mapping","SSO and group mapping",{"path":1500,"title":1501},"\u002Fdocs\u002Fexamples\u002Fkeycloak","Keycloak example on a laptop",{"path":1503,"title":1504},"\u002Fdocs\u002Fengines\u002Fclickhouse-ai-user","A ClickHouse user for the AI assistant",{"title":1506,"icon":1507,"intro":1508,"items":1509},"Install & operate","operate","Run it in production on Kubernetes, configure it, upgrade it and keep it healthy.",[1510,1513,1516,1519,1522,1525,1528],{"path":1511,"title":1512},"\u002Fdocs\u002Fdeploy\u002Fkubernetes","Kubernetes and Helm",{"path":1514,"title":1515},"\u002Fdocs\u002Fdeploy\u002Fkind","A company-like cluster on one computer (kind)",{"path":1517,"title":1518},"\u002Fdocs\u002Fdeploy\u002Fupgrading","Upgrading, backups and restore",{"path":1520,"title":1521},"\u002Fdocs\u002Fdeploy\u002Frelease-checklist","Install and upgrade checklist",{"path":1523,"title":1524},"\u002Fdocs\u002Fconfiguration","Configuration reference",{"path":1526,"title":1527},"\u002Fdocs\u002Fengine-settings","Settings from the panel",{"path":1529,"title":1530},"\u002Fdocs\u002Fdatabase-migrations","Database migrations",{"title":1532,"icon":1533,"intro":1534,"items":1535},"Reference","reference","Release process and the remaining reference pages.",[1536],{"path":1537,"title":1538},"\u002Fdocs\u002Frelease-process","Release process",1793656372083]