From 2b9f8d4e8c006b732c5ce00b763f990e48cc9f6a Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Mon, 22 Jan 2024 13:13:31 +0530 Subject: [PATCH 01/15] Add Qlik Cloud source code --- .../app/ingest/source/builder/constants.ts | 4 + .../app/ingest/source/builder/sources.json | 7 + datahub-web-react/src/images/qliklogo.png | Bin 0 -> 7505 bytes .../docs/sources/qlik-cloud/qlik_cloud_pre.md | 19 + .../sources/qlik-cloud/qlik_cloud_recipe.yml | 19 + metadata-ingestion/setup.py | 2 + .../ingestion/source/common/subtypes.py | 3 + .../ingestion/source/qlik_cloud/__init__.py | 0 .../ingestion/source/qlik_cloud/config.py | 90 +++++ .../source/qlik_cloud/data_classes.py | 128 +++++++ .../ingestion/source/qlik_cloud/qlik_api.py | 142 +++++++ .../ingestion/source/qlik_cloud/qlik_cloud.py | 346 ++++++++++++++++++ .../main/resources/boot/data_platforms.json | 10 + 13 files changed, 770 insertions(+) create mode 100644 datahub-web-react/src/images/qliklogo.png create mode 100644 metadata-ingestion/docs/sources/qlik-cloud/qlik_cloud_pre.md create mode 100644 metadata-ingestion/docs/sources/qlik-cloud/qlik_cloud_recipe.yml create mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/__init__.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/config.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/data_classes.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts index bd792d78856d5..6166f783635e8 100644 --- a/datahub-web-react/src/app/ingest/source/builder/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/constants.ts @@ -31,6 +31,7 @@ import mlflowLogo from '../../../../images/mlflowlogo.png'; import dynamodbLogo from '../../../../images/dynamodblogo.png'; import fivetranLogo from '../../../../images/fivetranlogo.png'; import csvLogo from '../../../../images/csv-logo.png'; +import qlikLogo from '../../../../images/qliklogo.png'; export const ATHENA = 'athena'; export const ATHENA_URN = `urn:li:dataPlatform:${ATHENA}`; @@ -113,6 +114,8 @@ export const FIVETRAN = 'fivetran'; export const FIVETRAN_URN = `urn:li:dataPlatform:${FIVETRAN}`; export const CSV = 'csv-enricher'; export const CSV_URN = `urn:li:dataPlatform:${CSV}`; +export const QLIKCLOUD = 'qlikcloud'; +export const QLIKCLOUD_URN = `urn:li:dataPlatform:${QLIKCLOUD}`; export const PLATFORM_URN_TO_LOGO = { [ATHENA_URN]: athenaLogo, @@ -149,6 +152,7 @@ export const PLATFORM_URN_TO_LOGO = { [VERTICA_URN]: verticaLogo, [FIVETRAN_URN]: fivetranLogo, [CSV_URN]: csvLogo, + [QLIKCLOUD_URN]: qlikLogo, }; export const SOURCE_TO_PLATFORM_URN = { diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index e33de13c100b7..2516c23d06ebf 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -236,5 +236,12 @@ "displayName": "Other", "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/", "recipe": "source:\n type: \n config:\n # Source-type specifics config\n " + }, + { + "urn": "urn:li:dataPlatform:qlikcloud", + "name": "qlikcloud", + "displayName": "QlikCloud", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/qlik-cloud/", + "recipe": "source:\n type: qlikcloud\n config:\n # Coordinates\n tenant_hostname: https://xyz12xz.us.qlikcloud.com\n # Coordinates\n api_key: QLIK_API_KEY\n\n # Optional - filter for certain space names instead of ingesting everything.\n # space_pattern:\n\n # allow:\n # - space_name\n # Whether personal space and apps and datasets should be ingested.\n extract_personal_entity: false\n ingest_owner: true" } ] diff --git a/datahub-web-react/src/images/qliklogo.png b/datahub-web-react/src/images/qliklogo.png new file mode 100644 index 0000000000000000000000000000000000000000..fe69faed8b9a9215c1f3e3d9fef26beb501ef789 GIT binary patch literal 7505 zcmYj$c{o(>|Gs^T#2{o#jBJ@p?(;Ua!%z^Iz9lV5SWD z!R#USSHn5w&UN|LRLzJ9*ipgNW44>An%6T#mx}Zkqa-GbsLlBN6I;^@Kqi%-9==sC zoT`bH?+IRb9ao0ccTUSo7x90%$t%$7p9_DClpZ%aY@ON<<1gL$at)EtA_SXHVG6N3 zcV*{I)trw!t6Lo`DEpoEce7lH0si7rXuO$fZ!9J&x6$Cx2NkAz%54~wVMh_2fY$ms z@l<N=nuWym}&GSme8drL8A{yr7UgKT;=bEJ%yua_@Rz-cg#3P(IPGrw$FGlpob^hviyi?$lm;2opc4q(5THWOmaENi|;4G`mXH*bg+w4%%7>PRBX{1GUXl=2sZ@uI1#fS!TWYFfl*f za7t?Ksq7Hq!sFkrXcqI8q}j$){+mKLUDwUDS%sfgrL{##)%=dnJVQyrvJr-!g}zQR zKunQH66g-F>u2Yb?zywZR5Qc?)l^~9uNL^ur#&)%;7_i>%QuD5BU zRWKVpw+s|fp_<)~q94S82$k;+<~G>LLcLqaIRQ-QiffDg202J^&D<@Nh^S57f`(rA zE|jdo3bdTHXajroHX~xA2h(DUXQ1A@ioU_udQJjE>l2|B||S(#98M{=3@^$tLs%CmN@!s7(aV4_Cq z`dX&Ef&78}I1eox_DxO2`%=I94V{&oCl5MdvcRUVCmxXDT~ngOV+68D#KrjRczl@m z>I(ci3V0iYPTo={o<`6}#8B76zodZ4c2n?#izi{?09M+_%SA|6040s(w% zLs%7-ECGBNfu*7x(7vacMmS9pQ5ZSvOiJlS7g99p^O(@=Sd`!_5bBmKMWI5K&)Wl) z!xGb28EL?+QqVneUvYHD(~F}1WN+y z+!*y04r0igK)DU01<6q5UHk9Us|1#m9Pq*P@{;CDPwM&z+}}e6#TbM_!+4gI0)QBvQndw(vtPIEo;vZD9O*BDbOqMS zEPl0aN5^ICJaEfPMts*f1)pqqs=l|$(ZytvEq+~6lQX?+4Lquzorr3TU6sW`q;l<* z)OEj39mGl56P1!+!}9HFS}{16(K8ymu0QQ|5t(fszO1z|mHs1j+3ag2&mruTIe^8q zsZ$$LB2k~?xeNvlIzIVgddcVtqIoPcV!m<{49Vh1Ic5hV(W)`?n~LLh--zyJIPKew zJF-VOBNx(={@e~X8F_2MZeTt`HQ(Wkn3vYP0A40pk^Z-!i3H`RxbKY;qA z8dN0>nr-EbNTkL|Q$(OoCuRhzh5pmopPo!UjHnGDxFqMo^Z?I6wyy?w=R@>lMp|5E zpkh|Y{iLRkCDGgRK^CCaY!(DfS8f#7%46EcWOm!t*4Z+GRQmnYyt0A*)Bk{2bHq1w z!sIKl-jGl=RNT|p|86iy5}@DTio@Pv=s1>JuSszm5P)6v;`v~y9SXd4_!awOtnxum zWuq>sEiXl%;ZOZz^^7EN5h%_P{ml#dt?=x9yfB3<&kYnmCN>?Y;-m5F?%9a`IA{n5 zU%-m3>r`iu8sqhN9>L1Ep&u>=o`79O0_l!aDaXE%3-K&@QN+lI!aDl&iQEXCqt>nx z;4hh)$00|p38jJt6j4OqNaZhdJ)9S$4?Cg@+;zk;<-~@pkpAHD(r&+?7qw{H=OkfS zN}UYjeF&Ov==?^H>~UAP!T;b4WSXL3jRERR9bZosN-_kWZcC19OBtGg?KH$VgW7K; zIn2j@0UlBI`v^3B$!S9ilJ&BEXLI{|7sn|f6LNGz9m9!Q*?jvq{Bxi1P__W$c)+FA zAH%{w){N*%J>7*aO<67n+4S4?Mh;4UKKJ$$ruCexZd;0FG)lPaDQLE3Hm8>63s>-D zrse(~EDG1u%XiHDEmOgx#ZR!>VlMBUCbD-}S8^=1w*cL!t+nBsMFx>8HJ*}7}b^6f7=B$19vdmZQU zjkbqo-?z&iCChEqgp4EWO$EN14wY~bh#I;z*%p`&!*VO(A;fy}RK7rYjQcgRfgxs< z<7pe_2I9QZhm9ZIHJ(j7^*O>p8DvfDEQ9ozGF?%(zbA~I^l91F7xaG}TfIxOEy{V_EoFWKN>+)!c%rKe+_k zCVnjm1<-eel^kfPKym_<`vvNS-1DNv;)^g3KPdhr_>|P3{l_dZ1izXDO?{}wj%17< z`!v!0&@$U35UpgrCW!xPqs2~$=fF+a4^#N~UpNc`sfGQv3wj|+psj~j2`jW5Q@jOV9b zbHufaG4?d3pJ=X8JIDvuBtAsYku5<|1m1H8=2Cd!>?4y;j1#*M{=bGn`)v~&I|mG& zK(%gsn~e{7>yy-_=B?k!t-Pio$9|QceP2!`(nocm$ljyro;|~^-d~ar_xW#e#oZ9U zgMRz4U@d-Y%0`JzuGo9rR|8sJ2-Woqd-h#9vz-z1Cz5EJdB5>BxBG*EnqJAo3Z9z8 zMl!8GzkbZe=eKZIi55=$0uXK^J>oFsiY7hVBdT`Z(x|u_Ox(XO7yhOPusAQM*f%d? z`gjaZ4GZ+e%Fu)1w?1WPSYDGK6ODjzx9>xp`St*f+;<;_rZPwVNP3KGKo2Ws}-?jFo41 zQzXRtmBC|QecE9pU}%dVL|dO@w2oq1UC;;V5D-6@NFKTf%hF_kB{&FGUfc>`bSf)Q zU{LXlag$7f{W*qt1p44S#@q$HEDKpn6_Z&;x>F&S%YQlJ_@%7OTcsE>bI&zj?)95q zJ{d1390^wVhTe7l80o;UqavDB)n>$7fLWq{beFYXXAw_2T zicF{?lUa#1p$ag1_NV{<(E#3;O+}f2oe2|AF~hPJe)sov61}x?#4Culg0F}F+lln; zZ&(`%uU+$F?|e`H9JJXB+ChPO*U$dh$uQ?()M4SN_J((iz{mIdV(+_b`p?Xt;*|=n z{M6ALlcA-OX{>yVbxz5LXuZz$!$*_pT`^}gwQ)Ud&R@21yYn5X_IDSmf4SdV(EC(9 zKwmGSt8|J|p)p2a2z99_Ei-Pxf>VEp8541CG`FM1-5PI^@JA;)Z^0tLMA-HBu#`6) z@eSDHok@UfZGgiap!;=~nW|$L4=>t;8(3*s9vxZ`%l*Ch=G!d z1W9(*TtWYPs0Jot33-qYNI8y37e7YJzkRHoV}rR@DW^Zks`P`S=pjGT?!F#i<;*~i zakFWk{355}G=-&NEHd% zJpIxjBOzA^P8s%FEYh{x4df>r^5zbhe<;>hc6e24d?w|+*Y_v6*Y{}aOmP)4#7p-2 z=n77W`tDEQ^cx!mHW~f2|H`EPB4yr>(W1NGXyDE#xl#TvuLPbq6Y6{@bu;U4LsS?l zb;eGXycWbQ)5&vQKAPLCZ1>=!Xl?`6)L-`=r%MApIgeay4jsLhV^VRywyVtI1<$uZ z0ekh>(E=YuMv15j=f;1G^RzkbSmT;&l%x{`Z#f42yT6~&b~yP24Q7B5G)QkRiKO69 z(5@{V2{WhNh@wuy_o;EMF#!YJeD!@Qp0#=dQwqTw3{_iZ=KhFnR2L4!57PeOrL@>B zdr9t$4Hz>zciy%fBS?En+7fK*DuDrp+=4hOjVdXepOIs3z8V-@+EMunB8GTvU|FgW z=z2_LAldgx_R{xQRCI~+i; zA;CJ{3V&rijxpW|C6<^sZHKEwBtUzX<*xAbW4k(^gLurq zv3Rh`FM|hrh_f=>kMCPvA~6Ce?UK05efi)Bp-0iad{6w*m*aCeu#D%H{bR~pIsH;p zgKJJ8H%il7Ghcf^*bTbQN?yq+o-!XUQrrd=Z8pC;7vRj&iQZyM$l9BM8-80J&D>^TGfDh%jk)j)!*JRNt z>HZ%6tCLYrD-CGjd=cy$@SXZPBn9r1Ut0?Kwh=eR^YY?lMKEK=$&0Cn4Rj1zZe5Kz z2sVAt~q|@hfp4yY5rAFo-*s*f74L|6Fi8SI5Fi#`wzpeQZdClnPr5w%WG%x{J7rn(I@- zUACOGU;b5nJE2J-a8JmN&?nog1V->Zih8m5<|>skv1>t#;q4jl8vcjEKD<$?zN9(x zY3cq}8yC%H@KUa&M0#$_bMdPqduJcTNblTMnKh^sWP4s|u-=yW${pak`k}#6TZ#)5 z^m*ct$ko;_je0i1(tENEN&uCUs=T~ay~{hyJ6-*FT|9SPf?Q!8w$_d`)qg}86 zxOK!${6|)I6n|~P4e{<&<2&70zj#lW4LhlTkvG!T=B6%cli6P<(DzuZwCMP)mUABb z%y-&|BXR9|3a$=v zv;iuPzr%Cl+v*A4qmZKoU~&BE6MZJo&-h~7$sBrj+buWuB=6^!rd;GjL) zV2F_xd@KZLy?^nJ89vDpc8*wiW}L!{Y~3Ga`@PV1}F<37KC zXXy2NDe?C&S06xVn^NyAwgQ=_+l-Hwgn>G0(06vF@&EB+(o`@?VS!D3ct+PKNKRG%A$TTBwPWYum?OmU~f7t?q2eF#Yn9;Xg zwW^=15I-STR}T4)wR9JhS95`1z_?HO^{rL%f>#p+MPg0igIWxo`hu3N3KQ=rqOtyI z5m4!?_r)t)5v~s`x>rz7;E-2&%rcDQ%f-gQRevifobTtH=Q?Q~m0UO`lXH_S+&?LG zb_;URF`#GMYy7jc*-ywREo&Lp{IRr!q-%iKq}kxPK*y9(*<;Gor!ZV3`m1*MEKE;{ z`gF`P#3*?cERZRdI#VKQLu|q)qSg4Ill_ zyw^^09{T2-NV@^FdL)0~*qboo^|{F&>uL~*?U3L=Wye~cf17<8x$*d>stncj)V9l$ z>;uelSed$42gHrSl&taxw1z&e+rSZGBq}r=4NwXYOi=$mm-Ua6=b7xbq+^03Bb0gj z6Us-Mv#3}8Es(W?Gr(G%dFyJ43FWN*+ez;GFEUg-q+dwXPnK32}p{%mKr0S)jBWWo%~e(XTNR( z+oxJ!XZEov<|8rJ(yhvys!~*kHD5}~%d+<6eGd(FFZ|HLqD$$P=M(7z`w_>HuOM~R z|97?hf2M8}@qgY6zSlL?%`!w?hew=(qW_otx{W@=mYKC*iwsEJv(083?SJa!umJl0 zE=ECbA%+5!F3U_zaJ0VW+vI4G5R-Q&bIi#>-G*h=)+r<PIG+n@Fq0%%>$~?MnBnJSiJH z?flPR>hLmcBioy1N?gG{2ZIlbx`w1|Y<4J)to!d*3}>J-C41NtpPZmcI}{gb8?%oU ze{7DSaz%32U_^BILQSXScCFwHTgt~{;`t%B7u;KzwE~Akf@Sw>GFAozh=JY~_3H3y95NM1=1-T}ruAAE)FF z)L@dYeR!M+iJ;5~2IO%v7*!2`cyoWj4_cy6?F{Tot(FtU|A$G4iSC=Ig31Lz5 za`@JkB$EqTLo4p}KVlnp+-i#(?H7N10?@%vWT-Wnz!Bf?^}oD<&bo*~iTYcILd=@1 zt&Hz!Qno#n5n*SWL*AI3NFv2qTwe9-8yLlz=Z}$v_a7DcwM6sojMg~TsHx(7g*X)p zViuxsI+(~jc{avdX_wKTepdI}dF?(=ddPGA)1OQVA*hjuVv|gRuXpSVAbIn6+kvt` zTMsCr!Dz@RN|?Y7y*LIbSbHE)*U3?5B#{y(FeVKONbd-T6X$KbX3&315livvV^3Y+ z@5Bk3y>drp&OYujY`-=nnQ9;tV_H2d%{D=gZgz2gvWn@(QH^w;1BjVF9Sd_Z3xv?# z8jcrQ;I!-IOJ?$qj2X$KgqNzowV7t6Lz;zzJUNqUAzYAsmmGJMeGzD{@&I*B;$r^Q zxO_qmXs>TqW}I=f=4-7*5o%bT`hjHnX(gGIv}lZ-C4J48=M@XfhKNhVqy=E6zvln; z0j^(;mvaeTiL}fmc!^4R9AGvHD3mF_Wamrqrf4S+VV8=lFQq0DoYL9zKPR&3*TnKo z#E)|5Y2s;NlaMs)^jd?1DZK@RJS~P>r;eToi`QZ2)9*6flgAlrFD(nN{+I$GYPQQS zJ%PjoYAq=g{`{ty1ZMqp5p@Sn<_=&k_X1`1qxP+7B-?o7s(bLG=Zc~KQI8h--}@RM-clwjISCISUM_VuiA-BuO#}MjD zqvBUeabs_6pQbr+{ass5Oucr<0Vfl0^ z<6DxvEL)6-Vvk4f87XS>w6&=J*?7&!@z!q++K8miy{)Sf@tRj}DD$$4A?o7CdI&2b zgxu*WSm;wofVlUgK*89)vi3Rg?T-P2k{;FH@sXN-j$3;b-Yx}r|*BL{_%I?{Lkg)j-~yd Qrd(Y1XPj(aoJQaKf2g%Mr2qf` literal 0 HcmV?d00001 diff --git a/metadata-ingestion/docs/sources/qlik-cloud/qlik_cloud_pre.md b/metadata-ingestion/docs/sources/qlik-cloud/qlik_cloud_pre.md new file mode 100644 index 0000000000000..328a50c9462c3 --- /dev/null +++ b/metadata-ingestion/docs/sources/qlik-cloud/qlik_cloud_pre.md @@ -0,0 +1,19 @@ +## Integration Details + +This source extracts the following: + +- Accessible spaces and apps within that spaces as Container. +- Qlik Datasets as Datahub Datasets with schema metadata. + +## Configuration Notes + +1. Refer [doc](https://qlik.dev/authenticate/api-key/generate-your-first-api-key/) to generate an API key from the hub. +2. Get tenant hostname from About tab after login to qlik sense account. + +## Concept mapping + +| Qlik Cloud | Datahub | +|--------------------------|--------------------------------------------------------------------------------------------------------| +| `Space` | [Container](https://datahubproject.io/docs/generated/metamodel/entities/container) | +| `App` | [Container](https://datahubproject.io/docs/generated/metamodel/entities/container) | +| `Dataset` | [Dataset](https://datahubproject.io/docs/generated/metamodel/entities/dataset) | \ No newline at end of file diff --git a/metadata-ingestion/docs/sources/qlik-cloud/qlik_cloud_recipe.yml b/metadata-ingestion/docs/sources/qlik-cloud/qlik_cloud_recipe.yml new file mode 100644 index 0000000000000..6975efcb9e48f --- /dev/null +++ b/metadata-ingestion/docs/sources/qlik-cloud/qlik_cloud_recipe.yml @@ -0,0 +1,19 @@ +source: + type: qlikcloud + config: + # Coordinates + tenant_hostname: "https://xyz12xz.us.qlikcloud.com" + # Credentials + api_key: "QLIK_API_KEY" + + # Optional - filter for certain space names instead of ingesting everything. + # space_pattern: + # allow: + # - space_name + + # Whether personal space and apps and datasets should be ingested. + extract_personal_entity: false + ingest_owner: true + +sink: + # sink configs diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index ec0bf076a0aff..da4052779a01e 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -397,6 +397,7 @@ # databricks is alias for unity-catalog and needs to be kept in sync "databricks": databricks | sql_common | sqllineage_lib, "fivetran": snowflake_common | bigquery_common, + "qlikcloud": {"requests"}, } # This is mainly used to exclude plugins from the Docker image. @@ -626,6 +627,7 @@ "gcs = datahub.ingestion.source.gcs.gcs_source:GCSSource", "sql-queries = datahub.ingestion.source.sql_queries:SqlQueriesSource", "fivetran = datahub.ingestion.source.fivetran.fivetran:FivetranSource", + "qlikcloud = datahub.ingestion.source.qlik_cloud.qlik_cloud:QlikCloudSource", ], "datahub.ingestion.transformer.plugins": [ "simple_remove_dataset_ownership = datahub.ingestion.transformer.remove_dataset_ownership:SimpleRemoveDatasetOwnership", diff --git a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py index 741b4789bef21..0720f9b33a802 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py @@ -15,6 +15,7 @@ class DatasetSubTypes(str, Enum): SALESFORCE_CUSTOM_OBJECT = "Custom Object" SALESFORCE_STANDARD_OBJECT = "Object" POWERBI_DATASET_TABLE = "PowerBI Dataset Table" + QLIK_DATASET = "Qlik Dataset" # TODO: Create separate entity... NOTEBOOK = "Notebook" @@ -39,6 +40,8 @@ class BIContainerSubTypes(str, Enum): TABLEAU_WORKBOOK = "Workbook" POWERBI_WORKSPACE = "Workspace" POWERBI_DATASET = "PowerBI Dataset" + QLIK_SPACE = "Qlik Space" + QLIK_APP = "Qlik App" class BIAssetSubTypes(str, Enum): diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/config.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/config.py new file mode 100644 index 0000000000000..abaa330808fb0 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/config.py @@ -0,0 +1,90 @@ +import logging +from dataclasses import dataclass +from typing import Optional + +import pydantic + +from datahub.configuration.common import AllowDenyPattern +from datahub.configuration.source_common import DatasetSourceConfigMixin +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalSourceReport, + StatefulStaleMetadataRemovalConfig, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionConfigBase, +) + +logger = logging.getLogger(__name__) + +QLIK_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + + +class Constant: + """ + keys used in qlik plugin + """ + + PLATFORM_NAME = "qlikcloud" + DATA = "data" + ID = "id" + NAME = "name" + DESCRIPTION = "description" + TYPE = "type" + OWNERID = "ownerId" + CREATEDAT = "createdAt" + UPDATEDAT = "updatedAt" + SECUREQRI = "secureQri" + QRI = "qri" + SPACEID = "spaceId" + CREATEDTIME = "createdTime" + LASTMODIFIEDTIME = "lastModifiedTime" + OPERATIONAL = "operational" + SIZE = "size" + ROWCOUNT = "rowCount" + DATATYPE = "dataType" + PRIMARYKEY = "primaryKey" + NULLABLE = "nullable" + SCHEMA = "schema" + DATAFIELDS = "dataFields" + RESOURCETYPE = "resourceType" + RESOURCEATTRIBUTES = "resourceAttributes" + USAGE = "usage" + CREATEDDATE = "createdDate" + MODIFIEDDATE = "modifiedDate" + RESOURCEID = "resourceId" + # Item type + APP = "app" + DATASET = "dataset" + # Personal entity constants + PERSONAL_SPACE_ID = "personal-space-id" + PERSONAL_SPACE_NAME = "personal_space" + + +@dataclass +class QlikSourceReport(StaleEntityRemovalSourceReport): + number_of_spaces: int = 0 + + def report_number_of_spaces(self, number_of_spaces: int) -> None: + self.number_of_spaces = number_of_spaces + + +class QlikSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin): + tenant_hostname: str = pydantic.Field(description="Qlik Tenant hostname") + api_key: str = pydantic.Field(description="Qlik API Key") + # Qlik space identifier + space_pattern: AllowDenyPattern = pydantic.Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns to filter Qlik spaces in ingestion", + ) + extract_personal_entity: Optional[bool] = pydantic.Field( + default=False, + description="Whether personal space and apps and datasets should be ingested.", + ) + ingest_owner: Optional[bool] = pydantic.Field( + default=True, + description="Ingest Owner from source. This will override Owner info entered from UI", + ) + # Configuration for stateful ingestion + stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = pydantic.Field( + default=None, description="Qlik Stateful Ingestion Config." + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/data_classes.py new file mode 100644 index 0000000000000..e740f88816687 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/data_classes.py @@ -0,0 +1,128 @@ +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from typing import Dict, List, Optional, Type, Union + +from datahub.emitter.mcp_builder import ContainerKey +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + BooleanType, + BytesType, + DateType, + NumberType, + StringType, + TimeType, +) + +FIELD_TYPE_MAPPING: Dict[ + str, + Type[ + Union[ + BooleanType, + BytesType, + DateType, + NumberType, + StringType, + TimeType, + ] + ], +] = { + "DATE": DateType, + "TIME": TimeType, + "DATETIME": DateType, + "TIMESTAMP": DateType, + "STRING": StringType, + "DOUBLE": NumberType, + "DECIMAL": NumberType, + "INTEGER": NumberType, + "BOOLEAN": BooleanType, + "BINARY": BytesType, +} + + +class SpaceKey(ContainerKey): + space: str + + +class AppKey(ContainerKey): + app: str + + +class SpaceType(Enum): + PERSONAL = "personal" + SHARED = "shared" + MANAGED = "managed" + DATA = "data" + + +@dataclass +class Space: + id: str + name: str + description: str + type: SpaceType + created_at: datetime + updated_at: datetime + owner_id: Optional[str] = None + + +@dataclass +class SchemaField: + name: str + data_type: str + primary_key: bool + nullable: bool + + +@dataclass +class Item: + id: str + name: str + qri: str + description: str + owner_id: str + space_id: str + created_at: datetime + updated_at: datetime + + +@dataclass +class App(Item): + usage: str + + +@dataclass +class QlikDataset(Item): + type: str + size: int + row_count: int + schema: List[SchemaField] + + +@dataclass +class User: + id: str + displayName: str + emailAddress: str + graphId: str + principalType: str + datasetUserAccessRight: Optional[str] = None + reportUserAccessRight: Optional[str] = None + dashboardUserAccessRight: Optional[str] = None + groupUserAccessRight: Optional[str] = None + + def get_urn_part(self, use_email: bool, remove_email_suffix: bool) -> str: + if use_email: + if remove_email_suffix: + return self.emailAddress.split("@")[0] + else: + return self.emailAddress + return f"users.{self.id}" + + def __members(self): + return (self.id,) + + def __eq__(self, instance): + return isinstance(instance, User) and self.__members() == instance.__members() + + def __hash__(self): + return hash(self.__members()) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py new file mode 100644 index 0000000000000..cdc00df8a2aa5 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py @@ -0,0 +1,142 @@ +import logging +import sys +from datetime import datetime +from typing import Any, List, Optional + +import requests + +from datahub.ingestion.source.qlik_cloud.config import ( + QLIK_DATETIME_FORMAT, + Constant, + QlikSourceConfig, +) +from datahub.ingestion.source.qlik_cloud.data_classes import ( + App, + Item, + QlikDataset, + SchemaField, + Space, +) + +# Logger instance +logger = logging.getLogger(__name__) + + +class QlikAPI: + def __init__(self, config: QlikSourceConfig) -> None: + self.config = config + self.session = requests.Session() + self.session.headers.update( + { + "Authorization": f"Bearer {self.config.api_key}", + "Content-Type": "application/json", + } + ) + self.base_url = f"{self.config.tenant_hostname}/api/v1" + + def log_http_error(self, message: str) -> Any: + logger.warning(message) + _, e, _ = sys.exc_info() + if isinstance(e, requests.exceptions.HTTPError): + logger.warning(f"HTTP status-code = {e.response.status_code}") + logger.debug(msg=message, exc_info=e) + return e + + def get_spaces(self) -> List[Space]: + spaces: List[Space] = [] + try: + response = self.session.get(f"{self.base_url}/spaces") + response.raise_for_status() + for space in response.json()[Constant.DATA]: + spaces.append( + Space( + id=space[Constant.ID], + name=space[Constant.NAME], + description=space[Constant.DESCRIPTION], + type=space[Constant.TYPE], + owner_id=space[Constant.OWNERID], + created_at=datetime.strptime( + space[Constant.CREATEDAT], QLIK_DATETIME_FORMAT + ), + updated_at=datetime.strptime( + space[Constant.UPDATEDAT], QLIK_DATETIME_FORMAT + ), + ) + ) + except Exception: + self.log_http_error(message="Unable to fetch spaces") + return spaces + + def _get_dataset(self, dataset_id: str) -> Optional[QlikDataset]: + try: + response = self.session.get(f"{self.base_url}/data-sets/{dataset_id}") + response.raise_for_status() + dataset = response.json() + return QlikDataset( + id=dataset[Constant.ID], + name=dataset[Constant.NAME], + qri=dataset[Constant.SECUREQRI], + description=dataset[Constant.DESCRIPTION], + space_id=dataset.get(Constant.SPACEID, ""), + owner_id=dataset[Constant.OWNERID], + created_at=datetime.strptime( + dataset[Constant.CREATEDTIME], QLIK_DATETIME_FORMAT + ), + updated_at=datetime.strptime( + dataset[Constant.LASTMODIFIEDTIME], QLIK_DATETIME_FORMAT + ), + type=dataset[Constant.TYPE], + size=dataset[Constant.OPERATIONAL].get(Constant.SIZE, 0), + row_count=dataset[Constant.OPERATIONAL][Constant.ROWCOUNT], + schema=[ + SchemaField( + name=field[Constant.NAME], + data_type=field[Constant.DATATYPE][Constant.TYPE], + primary_key=field[Constant.PRIMARYKEY], + nullable=field[Constant.NULLABLE], + ) + for field in dataset[Constant.SCHEMA][Constant.DATAFIELDS] + ], + ) + except Exception: + self.log_http_error(message="Unable to fetch items") + return None + + def get_items(self) -> List[Item]: + items: List[Item] = [] + try: + response = self.session.get(f"{self.base_url}/items") + response.raise_for_status() + data = response.json()[Constant.DATA] + for item in data: + resource_type = item[Constant.RESOURCETYPE] + resource_attributes = item[Constant.RESOURCEATTRIBUTES] + if resource_type == Constant.APP: + response = self.session.get(f"{self.base_url}/app/") + items.append( + App( + id=resource_attributes[Constant.ID], + name=resource_attributes[Constant.NAME], + qri=f"qri:app:sense://{resource_attributes[Constant.ID]}", + description=resource_attributes[Constant.DESCRIPTION], + space_id=resource_attributes[Constant.SPACEID], + usage=resource_attributes[Constant.USAGE], + owner_id=resource_attributes[Constant.OWNERID], + created_at=datetime.strptime( + resource_attributes[Constant.CREATEDDATE], + QLIK_DATETIME_FORMAT, + ), + updated_at=datetime.strptime( + resource_attributes[Constant.MODIFIEDDATE], + QLIK_DATETIME_FORMAT, + ), + ) + ) + elif resource_type == Constant.DATASET: + dataset = self._get_dataset(dataset_id=item[Constant.RESOURCEID]) + if dataset: + items.append(dataset) + + except Exception: + self.log_http_error(message="Unable to fetch items") + return items diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py new file mode 100644 index 0000000000000..5b3a5466bb745 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py @@ -0,0 +1,346 @@ +import logging +from datetime import datetime +from typing import Iterable, List, Optional + +from datahub.emitter.mce_builder import ( + make_data_platform_urn, + make_dataplatform_instance_urn, + make_dataset_urn_with_platform_instance, + make_user_urn, +) +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.mcp_builder import add_dataset_to_container, gen_containers +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.api.decorators import ( + SourceCapability, + SupportStatus, + capability, + config_class, + platform_name, + support_status, +) +from datahub.ingestion.api.source import ( + CapabilityReport, + MetadataWorkUnitProcessor, + SourceReport, + TestableSource, + TestConnectionReport, +) +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.common.subtypes import ( + BIContainerSubTypes, + DatasetSubTypes, +) +from datahub.ingestion.source.qlik_cloud.config import ( + Constant, + QlikSourceConfig, + QlikSourceReport, +) +from datahub.ingestion.source.qlik_cloud.data_classes import ( + FIELD_TYPE_MAPPING, + App, + AppKey, + QlikDataset, + SchemaField as QlikDatasetSchemaField, + Space, + SpaceKey, + SpaceType, +) +from datahub.ingestion.source.qlik_cloud.qlik_api import QlikAPI +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalHandler, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionSourceBase, +) +from datahub.metadata.com.linkedin.pegasus2avro.common import ( + Status, + SubTypes, + TimeStamp, +) +from datahub.metadata.com.linkedin.pegasus2avro.dataset import DatasetProperties +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + MySqlDDL, + NullType, + SchemaField, + SchemaMetadata, +) +from datahub.metadata.schema_classes import ( + DataPlatformInstanceClass, + OwnerClass, + OwnershipClass, + OwnershipTypeClass, + SchemaFieldDataTypeClass, +) + +# Logger instance +logger = logging.getLogger(__name__) + + +@platform_name("QlikCloud") +@config_class(QlikSourceConfig) +@support_status(SupportStatus.INCUBATING) +@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") +@capability( + SourceCapability.LINEAGE_COARSE, + "Enabled by default, configured using `extract_lineage`.", +) +class QlikCloudSource(StatefulIngestionSourceBase, TestableSource): + """ + This plugin extracts the following: + - Qlik Cloud Spaces, Apps, Datasets and Collections + - Names, descriptions and URLs of apps and datasets + """ + + config: QlikSourceConfig + reporter: QlikSourceReport + platform: str = "qlikcloud" + + def __init__(self, config: QlikSourceConfig, ctx: PipelineContext): + super(QlikCloudSource, self).__init__(config, ctx) + self.config = config + self.reporter = QlikSourceReport() + try: + self.qlik_api = QlikAPI(self.config) + except Exception as e: + logger.warning(e) + exit( + 1 + ) # Exit pipeline as we are not able to connect to Qlik Client Service. + + @staticmethod + def test_connection(config_dict: dict) -> TestConnectionReport: + test_report = TestConnectionReport() + try: + + test_report.basic_connectivity = CapabilityReport(capable=True) + except Exception as e: + test_report.basic_connectivity = CapabilityReport( + capable=False, failure_reason=str(e) + ) + return test_report + + @classmethod + def create(cls, config_dict, ctx): + config = QlikSourceConfig.parse_obj(config_dict) + return cls(config, ctx) + + def _gen_space_key(self, space_id: str) -> SpaceKey: + return SpaceKey( + space=space_id, + platform=self.platform, + instance=self.config.platform_instance, + ) + + def _gen_app_key(self, app_id: str) -> AppKey: + return AppKey( + app=app_id, + platform=self.platform, + instance=self.config.platform_instance, + ) + + def _gen_personal_space(self) -> Space: + return Space( + id=Constant.PERSONAL_SPACE_ID, + name=Constant.PERSONAL_SPACE_NAME, + description="", + type=SpaceType.PERSONAL, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + + def _get_allowed_spaces(self) -> List[Space]: + all_spaces = self.qlik_api.get_spaces() + allowed_spaces = [ + space + for space in all_spaces + if self.config.space_pattern.allowed(space.name) + ] + # Add personal space entity if flag is enable + if self.config.extract_personal_entity: + allowed_spaces.append(self._gen_personal_space()) + logger.info(f"Number of spaces = {len(allowed_spaces)}") + self.reporter.report_number_of_spaces(len(all_spaces)) + logger.info(f"Number of allowed spaces = {len(allowed_spaces)}") + return allowed_spaces + + def _gen_space_workunit(self, space: Space) -> Iterable[MetadataWorkUnit]: + yield from gen_containers( + container_key=self._gen_space_key(space.id), + name=space.name, + description=space.description, + sub_types=[BIContainerSubTypes.QLIK_SPACE], + extra_properties={Constant.TYPE: str(space.type)}, + owner_urn=make_user_urn(space.owner_id) + if self.config.ingest_owner and space.owner_id + else None, + created=int(space.created_at.timestamp() * 1000), + last_modified=int(space.updated_at.timestamp() * 1000), + ) + + def _gen_app_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: + yield from gen_containers( + container_key=self._gen_app_key(app.id), + name=app.name, + description=app.description, + sub_types=[BIContainerSubTypes.QLIK_APP], + parent_container_key=self._gen_space_key(app.space_id), + extra_properties={Constant.QRI: app.qri, Constant.USAGE: app.usage}, + owner_urn=make_user_urn(app.owner_id) if self.config.ingest_owner else None, + created=int(app.created_at.timestamp() * 1000), + last_modified=int(app.updated_at.timestamp() * 1000), + ) + + def _gen_dataset_urn(self, dataset_identifier: str) -> str: + return make_dataset_urn_with_platform_instance( + name=dataset_identifier, + env=self.config.env, + platform=self.platform, + platform_instance=self.config.platform_instance, + ) + + def gen_dataplatform_instance_aspect( + self, dataset_urn: str + ) -> Optional[MetadataWorkUnit]: + if self.config.platform_instance: + aspect = DataPlatformInstanceClass( + platform=make_data_platform_urn(self.platform), + instance=make_dataplatform_instance_urn( + self.platform, self.config.platform_instance + ), + ) + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=aspect + ).as_workunit() + else: + return None + + def _gen_dataset_owner_aspect( + self, dataset_urn: str, owner_id: str + ) -> MetadataWorkUnit: + aspect = OwnershipClass( + owners=[ + OwnerClass( + owner=make_user_urn(owner_id), type=OwnershipTypeClass.DATAOWNER + ) + ] + ) + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=aspect, + ).as_workunit() + + def _gen_schema_fields( + self, schema: List[QlikDatasetSchemaField] + ) -> List[SchemaField]: + schema_fields: List[SchemaField] = [] + for field in schema: + schema_field = SchemaField( + fieldPath=field.name, + type=SchemaFieldDataTypeClass( + type=FIELD_TYPE_MAPPING.get(field.data_type, NullType)() + ), + # NOTE: nativeDataType will not be in sync with older connector + nativeDataType=field.data_type, + nullable=field.nullable, + isPartOfKey=field.primary_key, + ) + schema_fields.append(schema_field) + return schema_fields + + def _gen_schema_metadata(self, dataset: QlikDataset) -> MetadataWorkUnit: + dataset_urn = self._gen_dataset_urn(dataset.id) + + schema_metadata = SchemaMetadata( + schemaName=dataset.id, + platform=make_data_platform_urn(self.platform), + version=0, + hash="", + platformSchema=MySqlDDL(tableSchema=""), + fields=self._gen_schema_fields(dataset.schema), + ) + + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=schema_metadata + ).as_workunit() + + def _gen_dataset_properties(self, dataset: QlikDataset) -> MetadataWorkUnit: + dataset_urn = self._gen_dataset_urn(dataset.id) + + dataset_properties = DatasetProperties( + name=dataset.name, + description=dataset.description, + qualifiedName=dataset.name, + created=TimeStamp(time=int(dataset.created_at.timestamp() * 1000)), + lastModified=TimeStamp(time=int(dataset.updated_at.timestamp() * 1000)), + ) + dataset_properties.customProperties.update( + { + Constant.QRI: dataset.qri, + Constant.SPACEID: dataset.space_id, + Constant.TYPE: dataset.type, + Constant.SIZE: str(dataset.size), + Constant.ROWCOUNT: str(dataset.row_count), + } + ) + + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=dataset_properties + ).as_workunit() + + def _gen_dataset_workunit(self, dataset: QlikDataset) -> Iterable[MetadataWorkUnit]: + dataset_urn = self._gen_dataset_urn(dataset.id) + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=Status(removed=False) + ).as_workunit() + + yield self._gen_schema_metadata(dataset) + + yield self._gen_dataset_properties(dataset) + + yield from add_dataset_to_container( + container_key=self._gen_space_key(dataset.space_id), + dataset_urn=dataset_urn, + ) + + dpi_aspect = self.gen_dataplatform_instance_aspect(dataset_urn) + if dpi_aspect: + yield dpi_aspect + + if self.config.ingest_owner: + yield self._gen_dataset_owner_aspect(dataset_urn, dataset.owner_id) + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=SubTypes(typeNames=[DatasetSubTypes.QLIK_DATASET]), + ).as_workunit() + + def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: + return [ + *super().get_workunit_processors(), + StaleEntityRemovalHandler.create( + self, self.config, self.ctx + ).workunit_processor, + ] + + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: + """ + Datahub Ingestion framework invoke this method + """ + logger.info("Qlik Cloud plugin execution is started") + for space in self._get_allowed_spaces(): + yield from self._gen_space_workunit(space) + for item in self.qlik_api.get_items(): + # If item is personal item and flag is Disable, skip ingesting personal item + if not item.space_id: + item.space_id = Constant.PERSONAL_SPACE_ID + if not self.config.extract_personal_entity: + continue + if isinstance(item, App): + yield from self._gen_app_workunit(item) + elif isinstance(item, QlikDataset): + yield from self._gen_dataset_workunit(item) + + def get_report(self) -> SourceReport: + return self.reporter diff --git a/metadata-service/war/src/main/resources/boot/data_platforms.json b/metadata-service/war/src/main/resources/boot/data_platforms.json index 0574f3fda4017..5fbd313c62188 100644 --- a/metadata-service/war/src/main/resources/boot/data_platforms.json +++ b/metadata-service/war/src/main/resources/boot/data_platforms.json @@ -584,5 +584,15 @@ "type": "OTHERS", "logoUrl": "/assets/platforms/csv-logo.png" } + }, + { + "urn": "urn:li:dataPlatform:qlikcloud", + "aspect": { + "datasetNameDelimiter": ".", + "name": "qlikcloud", + "displayName": "QlikCloud", + "type": "OTHERS", + "logoUrl": "/assets/platforms/qliklogo.png" + } } ] From aefbd4b62f9983c0dc1052bcbf7344961ccb3658 Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Mon, 22 Jan 2024 15:25:23 +0530 Subject: [PATCH 02/15] Modify qlik cloud soure name --- .../src/app/ingest/source/builder/constants.ts | 6 +++--- .../src/app/ingest/source/builder/sources.json | 8 ++++---- .../{qlik_cloud_pre.md => qlik-cloud_pre.md} | 0 ...qlik_cloud_recipe.yml => qlik-cloud_recipe.yml} | 2 +- metadata-ingestion/setup.py | 4 ++-- .../datahub/ingestion/source/qlik_cloud/config.py | 4 ++-- .../ingestion/source/qlik_cloud/qlik_api.py | 3 +++ .../ingestion/source/qlik_cloud/qlik_cloud.py | 14 ++++---------- .../src/main/resources/boot/data_platforms.json | 6 +++--- 9 files changed, 22 insertions(+), 25 deletions(-) rename metadata-ingestion/docs/sources/qlik-cloud/{qlik_cloud_pre.md => qlik-cloud_pre.md} (100%) rename metadata-ingestion/docs/sources/qlik-cloud/{qlik_cloud_recipe.yml => qlik-cloud_recipe.yml} (95%) diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts index 6166f783635e8..3c3a3e3263c76 100644 --- a/datahub-web-react/src/app/ingest/source/builder/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/constants.ts @@ -114,8 +114,8 @@ export const FIVETRAN = 'fivetran'; export const FIVETRAN_URN = `urn:li:dataPlatform:${FIVETRAN}`; export const CSV = 'csv-enricher'; export const CSV_URN = `urn:li:dataPlatform:${CSV}`; -export const QLIKCLOUD = 'qlikcloud'; -export const QLIKCLOUD_URN = `urn:li:dataPlatform:${QLIKCLOUD}`; +export const QLIK_CLOUD = 'qlik-cloud'; +export const QLIK_CLOUD_URN = `urn:li:dataPlatform:${QLIK_CLOUD}`; export const PLATFORM_URN_TO_LOGO = { [ATHENA_URN]: athenaLogo, @@ -152,7 +152,7 @@ export const PLATFORM_URN_TO_LOGO = { [VERTICA_URN]: verticaLogo, [FIVETRAN_URN]: fivetranLogo, [CSV_URN]: csvLogo, - [QLIKCLOUD_URN]: qlikLogo, + [QLIK_CLOUD_URN]: qlikLogo, }; export const SOURCE_TO_PLATFORM_URN = { diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index 2516c23d06ebf..89d6a855a330d 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -238,10 +238,10 @@ "recipe": "source:\n type: \n config:\n # Source-type specifics config\n " }, { - "urn": "urn:li:dataPlatform:qlikcloud", - "name": "qlikcloud", - "displayName": "QlikCloud", + "urn": "urn:li:dataPlatform:qlik-cloud", + "name": "qlik-cloud", + "displayName": "Qlik Cloud", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/qlik-cloud/", - "recipe": "source:\n type: qlikcloud\n config:\n # Coordinates\n tenant_hostname: https://xyz12xz.us.qlikcloud.com\n # Coordinates\n api_key: QLIK_API_KEY\n\n # Optional - filter for certain space names instead of ingesting everything.\n # space_pattern:\n\n # allow:\n # - space_name\n # Whether personal space and apps and datasets should be ingested.\n extract_personal_entity: false\n ingest_owner: true" + "recipe": "source:\n type: qlik-cloud\n config:\n # Coordinates\n tenant_hostname: https://xyz12xz.us.qlikcloud.com\n # Coordinates\n api_key: QLIK_API_KEY\n\n # Optional - filter for certain space names instead of ingesting everything.\n # space_pattern:\n\n # allow:\n # - space_name\n # Whether personal space and apps and datasets should be ingested.\n extract_personal_entity: false\n ingest_owner: true" } ] diff --git a/metadata-ingestion/docs/sources/qlik-cloud/qlik_cloud_pre.md b/metadata-ingestion/docs/sources/qlik-cloud/qlik-cloud_pre.md similarity index 100% rename from metadata-ingestion/docs/sources/qlik-cloud/qlik_cloud_pre.md rename to metadata-ingestion/docs/sources/qlik-cloud/qlik-cloud_pre.md diff --git a/metadata-ingestion/docs/sources/qlik-cloud/qlik_cloud_recipe.yml b/metadata-ingestion/docs/sources/qlik-cloud/qlik-cloud_recipe.yml similarity index 95% rename from metadata-ingestion/docs/sources/qlik-cloud/qlik_cloud_recipe.yml rename to metadata-ingestion/docs/sources/qlik-cloud/qlik-cloud_recipe.yml index 6975efcb9e48f..5d2e20b8b039c 100644 --- a/metadata-ingestion/docs/sources/qlik-cloud/qlik_cloud_recipe.yml +++ b/metadata-ingestion/docs/sources/qlik-cloud/qlik-cloud_recipe.yml @@ -1,5 +1,5 @@ source: - type: qlikcloud + type: qlik-cloud config: # Coordinates tenant_hostname: "https://xyz12xz.us.qlikcloud.com" diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index da4052779a01e..203da0f053bee 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -397,7 +397,7 @@ # databricks is alias for unity-catalog and needs to be kept in sync "databricks": databricks | sql_common | sqllineage_lib, "fivetran": snowflake_common | bigquery_common, - "qlikcloud": {"requests"}, + "qlik-cloud": {"requests"}, } # This is mainly used to exclude plugins from the Docker image. @@ -627,7 +627,7 @@ "gcs = datahub.ingestion.source.gcs.gcs_source:GCSSource", "sql-queries = datahub.ingestion.source.sql_queries:SqlQueriesSource", "fivetran = datahub.ingestion.source.fivetran.fivetran:FivetranSource", - "qlikcloud = datahub.ingestion.source.qlik_cloud.qlik_cloud:QlikCloudSource", + "qlik-cloud = datahub.ingestion.source.qlik_cloud.qlik_cloud:QlikCloudSource", ], "datahub.ingestion.transformer.plugins": [ "simple_remove_dataset_ownership = datahub.ingestion.transformer.remove_dataset_ownership:SimpleRemoveDatasetOwnership", diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/config.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/config.py index abaa330808fb0..17644d34b46d4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/config.py @@ -24,7 +24,7 @@ class Constant: keys used in qlik plugin """ - PLATFORM_NAME = "qlikcloud" + PLATFORM_NAME = "qlik-cloud" DATA = "data" ID = "id" NAME = "name" @@ -78,7 +78,7 @@ class QlikSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin): ) extract_personal_entity: Optional[bool] = pydantic.Field( default=False, - description="Whether personal space and apps and datasets should be ingested.", + description="Whether personal space, apps and datasets should be ingested.", ) ingest_owner: Optional[bool] = pydantic.Field( default=True, diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py index cdc00df8a2aa5..133929e64101e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py @@ -33,6 +33,9 @@ def __init__(self, config: QlikSourceConfig) -> None: } ) self.base_url = f"{self.config.tenant_hostname}/api/v1" + # Test connection by fetching list of api keys + logger.info("Trying to connect to {}".format(self.base_url)) + self.session.get(f"{self.base_url}/api-keys").raise_for_status() def log_http_error(self, message: str) -> Any: logger.warning(message) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py index 5b3a5466bb745..c61f9ab798cc5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py @@ -77,24 +77,18 @@ logger = logging.getLogger(__name__) -@platform_name("QlikCloud") +@platform_name("Qlik Cloud") @config_class(QlikSourceConfig) @support_status(SupportStatus.INCUBATING) @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") -@capability( - SourceCapability.LINEAGE_COARSE, - "Enabled by default, configured using `extract_lineage`.", -) class QlikCloudSource(StatefulIngestionSourceBase, TestableSource): """ - This plugin extracts the following: - - Qlik Cloud Spaces, Apps, Datasets and Collections - - Names, descriptions and URLs of apps and datasets + This plugin extracts Qlik Cloud Spaces, Apps, and Datasets """ config: QlikSourceConfig reporter: QlikSourceReport - platform: str = "qlikcloud" + platform: str = "qlik-cloud" def __init__(self, config: QlikSourceConfig, ctx: PipelineContext): super(QlikCloudSource, self).__init__(config, ctx) @@ -112,7 +106,7 @@ def __init__(self, config: QlikSourceConfig, ctx: PipelineContext): def test_connection(config_dict: dict) -> TestConnectionReport: test_report = TestConnectionReport() try: - + QlikAPI(QlikSourceConfig.parse_obj_allow_extras(config_dict)) test_report.basic_connectivity = CapabilityReport(capable=True) except Exception as e: test_report.basic_connectivity = CapabilityReport( diff --git a/metadata-service/war/src/main/resources/boot/data_platforms.json b/metadata-service/war/src/main/resources/boot/data_platforms.json index 5fbd313c62188..055771f71a409 100644 --- a/metadata-service/war/src/main/resources/boot/data_platforms.json +++ b/metadata-service/war/src/main/resources/boot/data_platforms.json @@ -586,11 +586,11 @@ } }, { - "urn": "urn:li:dataPlatform:qlikcloud", + "urn": "urn:li:dataPlatform:qlik-cloud", "aspect": { "datasetNameDelimiter": ".", - "name": "qlikcloud", - "displayName": "QlikCloud", + "name": "qlik-cloud", + "displayName": "Qlik Cloud", "type": "OTHERS", "logoUrl": "/assets/platforms/qliklogo.png" } From a3c46961a86c67448301cde7d5565b2ae3b6ee24 Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Mon, 22 Jan 2024 16:18:03 +0530 Subject: [PATCH 03/15] Add qlik source integration test cases --- .../ingestion/source/qlik_cloud/qlik_api.py | 1 - .../golden_test_platform_instace_ingest.json | 1091 +++++++++++++++++ .../golden_test_qlik_cloud_ingest.json | 997 +++++++++++++++ .../integration/qlik_cloud/test_qlik_cloud.py | 676 ++++++++++ 4 files changed, 2764 insertions(+), 1 deletion(-) create mode 100644 metadata-ingestion/tests/integration/qlik_cloud/golden_test_platform_instace_ingest.json create mode 100644 metadata-ingestion/tests/integration/qlik_cloud/golden_test_qlik_cloud_ingest.json create mode 100644 metadata-ingestion/tests/integration/qlik_cloud/test_qlik_cloud.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py index 133929e64101e..3f66aba322db3 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py @@ -115,7 +115,6 @@ def get_items(self) -> List[Item]: resource_type = item[Constant.RESOURCETYPE] resource_attributes = item[Constant.RESOURCEATTRIBUTES] if resource_type == Constant.APP: - response = self.session.get(f"{self.base_url}/app/") items.append( App( id=resource_attributes[Constant.ID], diff --git a/metadata-ingestion/tests/integration/qlik_cloud/golden_test_platform_instace_ingest.json b/metadata-ingestion/tests/integration/qlik_cloud/golden_test_platform_instace_ingest.json new file mode 100644 index 0000000000000..b84bf67c8178e --- /dev/null +++ b/metadata-ingestion/tests/integration/qlik_cloud/golden_test_platform_instace_ingest.json @@ -0,0 +1,1091 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-cloud", + "instance": "qlik_cloud_platform", + "space": "659d0e41d1b0ecce6eebc9b1", + "type": "shared" + }, + "name": "test_space", + "description": "", + "created": { + "time": 1704771818002 + }, + "lastModified": { + "time": 1704771818002 + } + } + }, + "systemMetadata": { + "lastObserved": 1705920242978, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1705920242980, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-cloud", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1705920242981, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Space" + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920242981, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1705920242982, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920242983, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-cloud", + "instance": "qlik_cloud_platform", + "space": "personal-space-id", + "type": "SpaceType.PERSONAL" + }, + "name": "personal_space", + "description": "", + "created": { + "time": 1705920242976 + }, + "lastModified": { + "time": 1705920242976 + } + } + }, + "systemMetadata": { + "lastObserved": 1705920242995, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1705920242997, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-cloud", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1705920242998, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Space" + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243000, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243001, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-cloud", + "instance": "qlik_cloud_platform", + "app": "b90c4d4e-0d07-4c24-9458-b17d1492660b", + "qri": "qri:app:sense://b90c4d4e-0d07-4c24-9458-b17d1492660b", + "usage": "ANALYTICS" + }, + "name": "test_app_1", + "description": "", + "created": { + "time": 1704769768928 + }, + "lastModified": { + "time": 1704771240100 + } + } + }, + "systemMetadata": { + "lastObserved": 1705920243003, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1705920243005, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-cloud", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1705920243007, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik App" + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243008, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1705920243008, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fdffbfd49b6139680de19df7d14a6737" + } + }, + "systemMetadata": { + "lastObserved": 1705920243010, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + }, + { + "id": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "urn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243011, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1705920243015, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "659d8aef92df266ef3aa5a7c", + "platform": "urn:li:dataPlatform:qlik-cloud", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Season", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243017, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "type": "DELIMETED", + "size": "38168", + "rowCount": "74" + }, + "name": "IPL_Matches_2022.csv", + "qualifiedName": "IPL_Matches_2022.csv", + "description": "", + "created": { + "time": 1704803735527 + }, + "lastModified": { + "time": 1704804512453 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1705920243020, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:1775d746acf71f0eea10452f850ff472" + } + }, + "systemMetadata": { + "lastObserved": 1705920243021, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-cloud", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1705920243022, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1705920243024, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243025, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + }, + { + "id": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "urn": "urn:li:container:1775d746acf71f0eea10452f850ff472" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243026, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-cloud", + "instance": "qlik_cloud_platform", + "app": "e09a68e7-18c9-461d-b957-043e0c045dcd", + "qri": "qri:app:sense://e09a68e7-18c9-461d-b957-043e0c045dcd", + "usage": "ANALYTICS" + }, + "name": "IPL_Matches_2022", + "description": "", + "created": { + "time": 1704803736545 + }, + "lastModified": { + "time": 1705297014684 + } + } + }, + "systemMetadata": { + "lastObserved": 1705920243027, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1705920243029, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-cloud", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1705920243030, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik App" + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243032, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1705920243033, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:1775d746acf71f0eea10452f850ff472" + } + }, + "systemMetadata": { + "lastObserved": 1705920243034, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + }, + { + "id": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "urn": "urn:li:container:1775d746acf71f0eea10452f850ff472" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243035, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1705920243037, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "65a137c849f82a37c625151b", + "platform": "urn:li:dataPlatform:qlik-cloud", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243038, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "qri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", + "spaceId": "personal-space-id", + "type": "CONNECTION_BASED_DATASET", + "size": "0", + "rowCount": "1" + }, + "name": "test_tabl", + "qualifiedName": "test_tabl", + "description": "", + "created": { + "time": 1705044592288 + }, + "lastModified": { + "time": 1705045296325 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1705920243039, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fdffbfd49b6139680de19df7d14a6737" + } + }, + "systemMetadata": { + "lastObserved": 1705920243040, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-cloud", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1705920243041, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1705920243042, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243042, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + }, + { + "id": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "urn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243043, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243044, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243045, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + }, + { + "id": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "urn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243046, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + }, + { + "id": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "urn": "urn:li:container:1775d746acf71f0eea10452f850ff472" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243046, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + }, + { + "id": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "urn": "urn:li:container:1775d746acf71f0eea10452f850ff472" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243047, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + }, + { + "id": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "urn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920243048, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/qlik_cloud/golden_test_qlik_cloud_ingest.json b/metadata-ingestion/tests/integration/qlik_cloud/golden_test_qlik_cloud_ingest.json new file mode 100644 index 0000000000000..dd9edd65061e0 --- /dev/null +++ b/metadata-ingestion/tests/integration/qlik_cloud/golden_test_qlik_cloud_ingest.json @@ -0,0 +1,997 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-cloud", + "space": "659d0e41d1b0ecce6eebc9b1", + "type": "shared" + }, + "name": "test_space", + "description": "", + "created": { + "time": 1704771818002 + }, + "lastModified": { + "time": 1704771818002 + } + } + }, + "systemMetadata": { + "lastObserved": 1705920115263, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1705920115264, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-cloud" + } + }, + "systemMetadata": { + "lastObserved": 1705920115265, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Space" + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115265, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1705920115266, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1705920115267, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-cloud", + "space": "personal-space-id", + "type": "SpaceType.PERSONAL" + }, + "name": "personal_space", + "description": "", + "created": { + "time": 1705920115262 + }, + "lastModified": { + "time": 1705920115262 + } + } + }, + "systemMetadata": { + "lastObserved": 1705920115272, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1705920115273, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-cloud" + } + }, + "systemMetadata": { + "lastObserved": 1705920115274, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Space" + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115274, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1705920115275, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-cloud", + "app": "b90c4d4e-0d07-4c24-9458-b17d1492660b", + "qri": "qri:app:sense://b90c4d4e-0d07-4c24-9458-b17d1492660b", + "usage": "ANALYTICS" + }, + "name": "test_app_1", + "description": "", + "created": { + "time": 1704769768928 + }, + "lastModified": { + "time": 1704771240100 + } + } + }, + "systemMetadata": { + "lastObserved": 1705920115276, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1705920115277, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-cloud" + } + }, + "systemMetadata": { + "lastObserved": 1705920115278, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik App" + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115279, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1705920115279, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:c48b664324e9c8852cc72ac953379725" + } + }, + "systemMetadata": { + "lastObserved": 1705920115280, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "urn": "urn:li:container:c48b664324e9c8852cc72ac953379725" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115281, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1705920115282, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "659d8aef92df266ef3aa5a7c", + "platform": "urn:li:dataPlatform:qlik-cloud", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Season", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115283, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "type": "DELIMETED", + "size": "38168", + "rowCount": "74" + }, + "name": "IPL_Matches_2022.csv", + "qualifiedName": "IPL_Matches_2022.csv", + "description": "", + "created": { + "time": 1704803735527 + }, + "lastModified": { + "time": 1704804512453 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1705920115287, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4" + } + }, + "systemMetadata": { + "lastObserved": 1705920115288, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1705920115288, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115289, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "urn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115290, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-cloud", + "app": "e09a68e7-18c9-461d-b957-043e0c045dcd", + "qri": "qri:app:sense://e09a68e7-18c9-461d-b957-043e0c045dcd", + "usage": "ANALYTICS" + }, + "name": "IPL_Matches_2022", + "description": "", + "created": { + "time": 1704803736545 + }, + "lastModified": { + "time": 1705297014684 + } + } + }, + "systemMetadata": { + "lastObserved": 1705920115291, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1705920115292, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-cloud" + } + }, + "systemMetadata": { + "lastObserved": 1705920115293, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik App" + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115293, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1705920115294, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4" + } + }, + "systemMetadata": { + "lastObserved": 1705920115295, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "urn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115295, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1705920115297, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "65a137c849f82a37c625151b", + "platform": "urn:li:dataPlatform:qlik-cloud", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115298, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "qri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", + "spaceId": "personal-space-id", + "type": "CONNECTION_BASED_DATASET", + "size": "0", + "rowCount": "1" + }, + "name": "test_tabl", + "qualifiedName": "test_tabl", + "description": "", + "created": { + "time": 1705044592288 + }, + "lastModified": { + "time": 1705045296325 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1705920115300, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:c48b664324e9c8852cc72ac953379725" + } + }, + "systemMetadata": { + "lastObserved": 1705920115301, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1705920115302, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115304, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "urn": "urn:li:container:c48b664324e9c8852cc72ac953379725" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115305, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1705920115306, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1705920115307, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "urn": "urn:li:container:c48b664324e9c8852cc72ac953379725" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115307, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "urn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115309, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "urn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115309, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "urn": "urn:li:container:c48b664324e9c8852cc72ac953379725" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1705920115310, + "runId": "qlik-cloud-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/qlik_cloud/test_qlik_cloud.py b/metadata-ingestion/tests/integration/qlik_cloud/test_qlik_cloud.py new file mode 100644 index 0000000000000..f8b5afa1e56ca --- /dev/null +++ b/metadata-ingestion/tests/integration/qlik_cloud/test_qlik_cloud.py @@ -0,0 +1,676 @@ +from typing import Any + +import pytest + +from datahub.ingestion.run.pipeline import Pipeline +from tests.test_helpers import mce_helpers + + +def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: + api_vs_response = { + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/api-keys": { + "method": "GET", + "status_code": 200, + "json": {}, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/spaces": { + "method": "GET", + "status_code": 200, + "json": { + "data": [ + { + "id": "659d0e41d1b0ecce6eebc9b1", + "type": "shared", + "ownerId": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "name": "test_space", + "description": "", + "meta": { + "actions": ["create", "delete", "read", "update"], + "roles": [], + "assignableRoles": [ + "codeveloper", + "consumer", + "dataconsumer", + "facilitator", + "producer", + ], + }, + "links": { + "self": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/spaces/659d0e41d1b0ecce6eebc9b1" + }, + "assignments": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/spaces/659d0e41d1b0ecce6eebc9b1/assignments" + }, + }, + "createdAt": "2024-01-09T09:13:38.002Z", + "createdBy": "657b5abe656297cec3d8b205", + "updatedAt": "2024-01-09T09:13:38.002Z", + } + ] + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items": { + "method": "GET", + "status_code": 200, + "json": { + "data": [ + { + "name": "test_app_1", + "thumbnailId": "", + "resourceAttributes": { + "_resourcetype": "app", + "createdDate": "2024-01-09T08:39:28.928Z", + "description": "", + "dynamicColor": "", + "hasSectionAccess": False, + "id": "b90c4d4e-0d07-4c24-9458-b17d1492660b", + "isDirectQueryMode": False, + "lastReloadTime": "2024-01-09T09:03:59.227Z", + "modifiedDate": "2024-01-09T09:04:00.100Z", + "name": "test_app_1", + "originAppId": "", + "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "ownerId": "657b5abe656297cec3d8b205", + "publishTime": "", + "published": False, + "spaceId": "", + "thumbnail": "", + "usage": "ANALYTICS", + }, + "resourceCustomAttributes": None, + "resourceUpdatedAt": "2024-01-09T09:04:00Z", + "resourceType": "app", + "resourceId": "b90c4d4e-0d07-4c24-9458-b17d1492660b", + "resourceCreatedAt": "2024-01-09T08:39:28Z", + "id": "659d064133e7d51dbcbb5911", + "createdAt": "2024-01-09T08:39:29Z", + "updatedAt": "2024-01-09T09:04:00Z", + "creatorId": "657b5abe656297cec3d8b205", + "updaterId": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "isFavorited": False, + "links": { + "self": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/659d064133e7d51dbcbb5911" + }, + "collections": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/659d064133e7d51dbcbb5911/collections" + }, + "open": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/b90c4d4e-0d07-4c24-9458-b17d1492660b" + }, + }, + "actions": [ + "change_owner", + "change_space", + "create", + "create_session_app", + "delete", + "delete_share", + "duplicate", + "export", + "export_reduced", + "exportappdata", + "import", + "read", + "reload", + "source", + "update", + ], + "collectionIds": ["659d0c6dad4c8e1b1e1f6ebe"], + "meta": { + "isFavorited": False, + "actions": [ + "change_owner", + "change_space", + "create", + "create_session_app", + "delete", + "delete_share", + "duplicate", + "export", + "export_reduced", + "exportappdata", + "import", + "read", + "reload", + "source", + "update", + ], + "tags": [], + "collections": [ + { + "id": "659d0c6dad4c8e1b1e1f6ebe", + "name": "test_collection", + } + ], + }, + "ownerId": "657b5abe656297cec3d8b205", + "resourceReloadEndTime": "2024-01-09T09:03:59Z", + "resourceReloadStatus": "ok", + "resourceSize": {"appFile": 196608, "appMemory": 625}, + "itemViews": {"total": 10, "trend": 0.5, "unique": 2}, + }, + { + "name": "IPL_Matches_2022.csv", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "resourceAttributes": { + "appType": "QIX-DF", + "dataStoreName": "DataFilesStore", + "dataStoreType": "qix-datafiles", + "qri": "qdf:qix-datafiles:ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A:sid@659d0e41d1b0ecce6eebc9b1:IPL_Matches_2022.csv", + "secureQri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", + "sourceSystemId": "QIX-DF_6fb35d21-24d0-474b-bf8b-536c6e9dc717", + "technicalDescription": "", + "technicalName": "IPL_Matches_2022.csv", + "type": "DELIMETED", + "version": "2", + }, + "resourceCustomAttributes": None, + "resourceUpdatedAt": "2024-01-09T18:18:32Z", + "resourceType": "dataset", + "resourceSubType": "qix-df", + "resourceId": "659d8aef92df266ef3aa5a7c", + "resourceCreatedAt": "2024-01-09T18:05:35Z", + "id": "659d8aef12794f37026cb262", + "createdAt": "2024-01-09T18:05:35Z", + "updatedAt": "2024-01-09T18:18:32Z", + "creatorId": "657b5abe656297cec3d8b205", + "updaterId": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "isFavorited": False, + "links": { + "self": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/659d8aef12794f37026cb262" + }, + "collections": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/659d8aef12794f37026cb262/collections" + }, + }, + "actions": [ + "create", + "delete", + "list", + "profile", + "read", + "update", + ], + "collectionIds": [], + "meta": { + "isFavorited": False, + "actions": [ + "create", + "delete", + "list", + "profile", + "read", + "update", + ], + "tags": [], + "collections": [], + }, + "ownerId": "657b5abe656297cec3d8b205", + "resourceReloadEndTime": "", + "resourceReloadStatus": "", + "resourceSize": {"appFile": 0, "appMemory": 0}, + "itemViews": {}, + }, + { + "name": "IPL_Matches_2022", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "thumbnailId": "", + "resourceAttributes": { + "_resourcetype": "app", + "createdDate": "2024-01-09T18:05:36.545Z", + "description": "", + "dynamicColor": "", + "hasSectionAccess": False, + "id": "e09a68e7-18c9-461d-b957-043e0c045dcd", + "isDirectQueryMode": False, + "lastReloadTime": "2024-01-15T11:06:53.070Z", + "modifiedDate": "2024-01-15T11:06:54.684Z", + "name": "IPL_Matches_2022", + "originAppId": "", + "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "ownerId": "657b5abe656297cec3d8b205", + "publishTime": "", + "published": False, + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "thumbnail": "", + "usage": "ANALYTICS", + }, + "resourceCustomAttributes": None, + "resourceUpdatedAt": "2024-01-15T11:06:54Z", + "resourceType": "app", + "resourceId": "e09a68e7-18c9-461d-b957-043e0c045dcd", + "resourceCreatedAt": "2024-01-09T18:05:36Z", + "id": "659d8af1ef4eadeead3ec0ec", + "createdAt": "2024-01-09T18:05:37Z", + "updatedAt": "2024-01-15T11:06:54Z", + "creatorId": "657b5abe656297cec3d8b205", + "updaterId": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "isFavorited": False, + "links": { + "self": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/659d8af1ef4eadeead3ec0ec" + }, + "collections": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/659d8af1ef4eadeead3ec0ec/collections" + }, + "open": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/e09a68e7-18c9-461d-b957-043e0c045dcd" + }, + }, + "actions": [ + "change_owner", + "change_space", + "create", + "delete", + "delete_share", + "duplicate", + "export", + "export_reduced", + "exportappdata", + "import", + "read", + "reload", + "share", + "source", + "update", + ], + "collectionIds": [], + "meta": { + "isFavorited": False, + "actions": [ + "change_owner", + "change_space", + "create", + "delete", + "delete_share", + "duplicate", + "export", + "export_reduced", + "exportappdata", + "import", + "read", + "reload", + "share", + "source", + "update", + ], + "tags": [], + "collections": [], + }, + "ownerId": "657b5abe656297cec3d8b205", + "resourceReloadEndTime": "2024-01-15T11:06:53Z", + "resourceReloadStatus": "ok", + "resourceSize": {"appFile": 17634, "appMemory": 36315}, + "itemViews": {"total": 4, "trend": 0.5, "unique": 2}, + }, + { + "name": "test_tabl", + "resourceAttributes": { + "appType": "CONNECTION_BASED_DATASET", + "dataStoreName": "gbqqri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#QgUSMctzlGjr47d2tMP35YlS93h78jVECRqtByqxPEE", + "dataStoreType": "gbq", + "qri": "a4d1b9ef-e629-4943-809a-2713aa3a5345", + "secureQri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", + "sourceSystemId": "", + "technicalDescription": "", + "technicalName": "harshal-playground-306419'.'test_dataset'.'test_table", + "type": "CONNECTION_BASED_DATASET", + "version": "2", + }, + "resourceCustomAttributes": None, + "resourceUpdatedAt": "2024-01-12T13:11:36Z", + "resourceType": "dataset", + "resourceSubType": "connection_based_dataset", + "resourceId": "65a137c849f82a37c625151b", + "resourceCreatedAt": "2024-01-12T12:59:52Z", + "id": "65a137c8d5a03b02d359624a", + "createdAt": "2024-01-12T12:59:52Z", + "updatedAt": "2024-01-12T13:11:36Z", + "creatorId": "657b5abe656297cec3d8b205", + "updaterId": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "isFavorited": False, + "links": { + "self": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/65a137c8d5a03b02d359624a" + }, + "collections": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/65a137c8d5a03b02d359624a/collections" + }, + }, + "actions": [ + "create", + "delete", + "list", + "profile", + "read", + "update", + ], + "collectionIds": [], + "meta": { + "isFavorited": False, + "actions": [ + "create", + "delete", + "list", + "profile", + "read", + "update", + ], + "tags": [], + "collections": [], + }, + "ownerId": "657b5abe656297cec3d8b205", + "resourceReloadEndTime": "", + "resourceReloadStatus": "", + "resourceSize": {"appFile": 0, "appMemory": 0}, + "itemViews": { + "total": 2, + "trend": 0.3, + "unique": 1, + "usedBy": 1, + }, + }, + ], + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/data-sets/659d8aef92df266ef3aa5a7c": { + "method": "GET", + "status_code": 200, + "json": { + "id": "659d8aef92df266ef3aa5a7c", + "name": "IPL_Matches_2022.csv", + "description": "", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "ownerId": "657b5abe656297cec3d8b205", + "createdTime": "2024-01-09T18:05:35.527Z", + "createdBy": "657b5abe656297cec3d8b205", + "lastModifiedTime": "2024-01-09T18:18:32.453Z", + "lastModifiedBy": "657b5abe656297cec3d8b205", + "version": 2, + "technicalDescription": "", + "technicalName": "IPL_Matches_2022.csv", + "properties": {"ModifiedByProfileService": False}, + "dataAssetInfo": { + "id": "659d8aef92df266ef3aa5a7b", + "name": "test_space", + "dataStoreInfo": { + "id": "659d8aef92df266ef3aa5a7a", + "name": "DataFilesStore", + "type": "qix-datafiles", + }, + }, + "operational": { + "size": 38168, + "rowCount": 74, + "lastLoadTime": "2024-01-09T18:05:35.527Z", + "logMessage": '{\n "cloudEventsVersion": "0.1",\n "source": "com.qlik/qix-datafiles",\n "contentType": "application/json",\n "eventID": "48c935d3-499c-4333-96e6-c8922af6e238",\n "eventType": "com.qlik.datafile.created",\n "eventTypeVersion": "0.0.1",\n "eventTime": "2024-01-09T18:05:35.834061Z",\n "extensions": {\n "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A",\n "userId": "657b5abe656297cec3d8b205",\n "header": {\n "traceparent": [\n "00-000000000000000063d931098d8f607c-7f2bea16c07c1c3a-01"\n ]\n }\n },\n "data": {\n "id": "6fb35d21-24d0-474b-bf8b-536c6e9dc717",\n "name": "IPL_Matches_2022.csv",\n "createdDate": "2024-01-09T18:05:35.5279392Z",\n "modifiedDate": "2024-01-09T18:05:35.828813Z",\n "createdByUser": "657b5abe656297cec3d8b205",\n "modifiedByUser": "657b5abe656297cec3d8b205",\n "ownerId": "657b5abe656297cec3d8b205",\n "spaceId": "659d0e41d1b0ecce6eebc9b1",\n "size": 38168,\n "contentUpdated": true,\n "isInternal": false,\n "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4"\n },\n "messageTopic": "system-events.datafiles/Created/ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A/6fb35d21-24d0-474b-bf8b-536c6e9dc717"\n}', + "status": "com.qlik.datafile.created", + "location": "48c935d3-499c-4333-96e6-c8922af6e238", + "contentUpdated": True, + "lastUpdateTime": "2024-01-09T18:05:35.828Z", + }, + "schema": { + "dataFields": [ + { + "name": "ID", + "index": 0, + "dataType": { + "type": "INTEGER", + "properties": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + }, + "tags": ["$integer", "$numeric"], + "encrypted": False, + "primaryKey": False, + "sensitive": False, + "orphan": False, + "nullable": False, + }, + { + "name": "City", + "index": 1, + "dataType": { + "type": "STRING", + "properties": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + }, + "tags": ["$text", "$ascii"], + "encrypted": False, + "primaryKey": False, + "sensitive": False, + "orphan": False, + "nullable": False, + }, + { + "name": "Date", + "index": 2, + "dataType": { + "type": "DATE", + "properties": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + }, + "tags": ["$integer", "$numeric", "$date", "$timestamp"], + "encrypted": False, + "primaryKey": False, + "sensitive": False, + "orphan": False, + "nullable": False, + }, + { + "name": "Season", + "index": 3, + "dataType": { + "type": "INTEGER", + "properties": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + }, + "tags": ["$integer", "$numeric"], + "encrypted": False, + "primaryKey": False, + "sensitive": False, + "orphan": False, + "nullable": False, + }, + ], + "loadOptions": { + "qDataFormat": { + "qType": "CSV", + "qLabel": "embedded labels", + "qQuote": "msq", + "qDelimiter": { + "qName": "Comma", + "qScriptCode": "','", + "qNumber": 44, + }, + "qCodePage": 28591, + "qHeaderSize": 0, + "qRecordSize": 0, + "qTabSize": 0, + } + }, + "effectiveDate": "2024-01-09T18:05:39.713Z", + "overrideSchemaAnomalies": False, + }, + "qri": "qdf:qix-datafiles:ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A:sid@659d0e41d1b0ecce6eebc9b1:IPL_Matches_2022.csv", + "secureQri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", + "type": "DELIMETED", + "classifications": { + "personalInformation": [], + "sensitiveInformation": [], + }, + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/data-sets/65a137c849f82a37c625151b": { + "method": "GET", + "status_code": 200, + "json": { + "id": "65a137c849f82a37c625151b", + "name": "test_tabl", + "description": "", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "ownerId": "657b5abe656297cec3d8b205", + "createdTime": "2024-01-12T12:59:52.288Z", + "createdBy": "657b5abe656297cec3d8b205", + "lastModifiedTime": "2024-01-12T13:11:36.325Z", + "lastModifiedBy": "657b5abe656297cec3d8b205", + "version": 2, + "technicalDescription": "", + "technicalName": "harshal-playground-306419'.'test_dataset'.'test_table", + "properties": {"ModifiedByProfileService": False}, + "dataAssetInfo": { + "id": "65a137c849f82a37c625151a", + "name": "gbqqri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#QgUSMctzlGjr47d2tMP35YlS93h78jVECRqtByqxPEE", + "dataStoreInfo": { + "id": "65a137c849f82a37c6251519", + "name": "gbqqri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#QgUSMctzlGjr47d2tMP35YlS93h78jVECRqtByqxPEE", + "type": "gbq", + }, + }, + "operational": { + "rowCount": 1, + "contentUpdated": False, + "tableConnectionInfo": { + "tableName": "test_table", + "selectionScript": "[test_table]:\nSELECT name\nFROM `harshal-playground-306419`.`test_dataset`.`test_table`;", + "additionalProperties": { + "fields": '[{"name":"name","fullName":"name","nativeType":"STRING","nativeFieldInfo":{"dataType":12,"name":"name","nullable":1,"ordinalPostion":1,"scale":0,"size":65535,"typeName":"STRING"},"customProperties":[],"isSelected":true}]', + "tableRequestParameters": '[{\\"name\\":\\"database\\",\\"value\\":\\"harshal-playground-306419\\"},{\\"name\\":\\"owner\\",\\"value\\":\\"test_dataset\\"}]', + }, + }, + }, + "schema": { + "dataFields": [ + { + "name": "name", + "index": 0, + "dataType": { + "type": "STRING", + "properties": {"qType": "A", "qnDec": 0, "qUseThou": 0}, + }, + "tags": ["$text", "$ascii"], + "encrypted": False, + "primaryKey": False, + "sensitive": False, + "orphan": False, + "nullable": False, + } + ], + "loadOptions": {}, + "effectiveDate": "2024-01-12T12:59:54.522Z", + "anomalies": ["$warning-single-text-column"], + "overrideSchemaAnomalies": False, + }, + "qri": "a4d1b9ef-e629-4943-809a-2713aa3a5345", + "secureQri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", + "type": "CONNECTION_BASED_DATASET", + "classifications": { + "personalInformation": [], + "sensitiveInformation": [], + }, + }, + }, + } + + api_vs_response.update(override_data) + + for url in api_vs_response.keys(): + request_mock.register_uri( + api_vs_response[url]["method"], + url, + json=api_vs_response[url]["json"], + status_code=api_vs_response[url]["status_code"], + ) + + +def default_config(): + return { + "tenant_hostname": "https://iq37k6byr9lgam8.us.qlikcloud.com", + "api_key": "qlik-api-key", + "space_pattern": { + "allow": [ + "test_space", + ] + }, + "extract_personal_entity": True, + } + + +@pytest.mark.integration +def test_qlik_cloud_ingest(pytestconfig, tmp_path, requests_mock): + + test_resources_dir = pytestconfig.rootpath / "tests/integration/qlik_cloud" + + register_mock_api(request_mock=requests_mock) + + output_path: str = f"{tmp_path}/qlik_cloud_ingest_mces.json" + + pipeline = Pipeline.create( + { + "run_id": "qlik-cloud-test", + "source": { + "type": "qlik-cloud", + "config": { + **default_config(), + }, + }, + "sink": { + "type": "file", + "config": { + "filename": output_path, + }, + }, + } + ) + + pipeline.run() + pipeline.raise_from_status() + golden_file = "golden_test_qlik_cloud_ingest.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=output_path, + golden_path=f"{test_resources_dir}/{golden_file}", + ) + + +@pytest.mark.integration +def test_platform_instance_ingest(pytestconfig, tmp_path, requests_mock): + + test_resources_dir = pytestconfig.rootpath / "tests/integration/qlik_cloud" + + register_mock_api(request_mock=requests_mock) + + output_path: str = f"{tmp_path}/qlik_platform_instace_ingest_mces.json" + + pipeline = Pipeline.create( + { + "run_id": "qlik-cloud-test", + "source": { + "type": "qlik-cloud", + "config": { + **default_config(), + "platform_instance": "qlik_cloud_platform", + }, + }, + "sink": { + "type": "file", + "config": { + "filename": output_path, + }, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() + golden_file = "golden_test_platform_instace_ingest.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=output_path, + golden_path=f"{test_resources_dir}/{golden_file}", + ) From d7c9a3143c0b330be804b30eb52acb940c1b8d92 Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Mon, 22 Jan 2024 17:26:26 +0530 Subject: [PATCH 04/15] Add missing capabilities decorator --- .../src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py index c61f9ab798cc5..a7e5fe0fbbcc8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py @@ -80,7 +80,12 @@ @platform_name("Qlik Cloud") @config_class(QlikSourceConfig) @support_status(SupportStatus.INCUBATING) +@capability(SourceCapability.DESCRIPTIONS, "Enabled by default") @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") +@capability( + SourceCapability.OWNERSHIP, + "Enabled by default, configured using `ingest_owner`", +) class QlikCloudSource(StatefulIngestionSourceBase, TestableSource): """ This plugin extracts Qlik Cloud Spaces, Apps, and Datasets From f98b5344b99280b3917e80d3191e52c03c1a2723 Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Tue, 30 Jan 2024 20:57:42 +0530 Subject: [PATCH 05/15] Address review comments and add dashboard and chart metadata ingestion code --- .../app/ingest/source/builder/constants.ts | 6 +- .../app/ingest/source/builder/sources.json | 10 +- .../docs/sources/qlik-cloud/qlik-cloud_pre.md | 19 - .../docs/sources/qlik-sense/qlik-sense_pre.md | 23 + .../qlik-sense_recipe.yml} | 2 +- metadata-ingestion/setup.py | 4 +- .../source/qlik_cloud/data_classes.py | 128 ---- .../ingestion/source/qlik_cloud/qlik_api.py | 144 ---- .../{qlik_cloud => qlik_sense}/__init__.py | 0 .../{qlik_cloud => qlik_sense}/config.py | 45 +- .../source/qlik_sense/data_classes.py | 182 +++++ .../ingestion/source/qlik_sense/qlik_api.py | 196 ++++++ .../qlik_sense.py} | 260 +++++--- .../golden_test_platform_instace_ingest.json | 630 ++++++++++++------ .../golden_test_qlik_sense_ingest.json} | 521 ++++++++++----- .../test_qlik_sense.py} | 200 +++++- .../main/resources/boot/data_platforms.json | 6 +- 17 files changed, 1595 insertions(+), 781 deletions(-) delete mode 100644 metadata-ingestion/docs/sources/qlik-cloud/qlik-cloud_pre.md create mode 100644 metadata-ingestion/docs/sources/qlik-sense/qlik-sense_pre.md rename metadata-ingestion/docs/sources/{qlik-cloud/qlik-cloud_recipe.yml => qlik-sense/qlik-sense_recipe.yml} (95%) delete mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/data_classes.py delete mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py rename metadata-ingestion/src/datahub/ingestion/source/{qlik_cloud => qlik_sense}/__init__.py (100%) rename metadata-ingestion/src/datahub/ingestion/source/{qlik_cloud => qlik_sense}/config.py (73%) create mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py rename metadata-ingestion/src/datahub/ingestion/source/{qlik_cloud/qlik_cloud.py => qlik_sense/qlik_sense.py} (55%) rename metadata-ingestion/tests/integration/{qlik_cloud => qlik_sense}/golden_test_platform_instace_ingest.json (54%) rename metadata-ingestion/tests/integration/{qlik_cloud/golden_test_qlik_cloud_ingest.json => qlik_sense/golden_test_qlik_sense_ingest.json} (57%) rename metadata-ingestion/tests/integration/{qlik_cloud/test_qlik_cloud.py => qlik_sense/test_qlik_sense.py} (82%) diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts index 3c3a3e3263c76..1a0aa9b9e6957 100644 --- a/datahub-web-react/src/app/ingest/source/builder/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/constants.ts @@ -114,8 +114,8 @@ export const FIVETRAN = 'fivetran'; export const FIVETRAN_URN = `urn:li:dataPlatform:${FIVETRAN}`; export const CSV = 'csv-enricher'; export const CSV_URN = `urn:li:dataPlatform:${CSV}`; -export const QLIK_CLOUD = 'qlik-cloud'; -export const QLIK_CLOUD_URN = `urn:li:dataPlatform:${QLIK_CLOUD}`; +export const QLIK_SENSE = 'qlik-sense'; +export const QLIK_SENSE_URN = `urn:li:dataPlatform:${QLIK_SENSE}`; export const PLATFORM_URN_TO_LOGO = { [ATHENA_URN]: athenaLogo, @@ -152,7 +152,7 @@ export const PLATFORM_URN_TO_LOGO = { [VERTICA_URN]: verticaLogo, [FIVETRAN_URN]: fivetranLogo, [CSV_URN]: csvLogo, - [QLIK_CLOUD_URN]: qlikLogo, + [QLIK_SENSE_URN]: qlikLogo, }; export const SOURCE_TO_PLATFORM_URN = { diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index 89d6a855a330d..b18d4e7222741 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -238,10 +238,10 @@ "recipe": "source:\n type: \n config:\n # Source-type specifics config\n " }, { - "urn": "urn:li:dataPlatform:qlik-cloud", - "name": "qlik-cloud", - "displayName": "Qlik Cloud", - "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/qlik-cloud/", - "recipe": "source:\n type: qlik-cloud\n config:\n # Coordinates\n tenant_hostname: https://xyz12xz.us.qlikcloud.com\n # Coordinates\n api_key: QLIK_API_KEY\n\n # Optional - filter for certain space names instead of ingesting everything.\n # space_pattern:\n\n # allow:\n # - space_name\n # Whether personal space and apps and datasets should be ingested.\n extract_personal_entity: false\n ingest_owner: true" + "urn": "urn:li:dataPlatform:qlik-sense", + "name": "qlik-sense", + "displayName": "Qlik Sense", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/qlik-sense/", + "recipe": "source:\n type: qlik-sense\n config:\n # Coordinates\n tenant_hostname: https://xyz12xz.us.qlikcloud.com\n # Coordinates\n api_key: QLIK_API_KEY\n\n # Optional - filter for certain space names instead of ingesting everything.\n # space_pattern:\n\n # allow:\n # - space_name\n # Whether personal space and apps and datasets should be ingested.\n extract_personal_entity: false\n ingest_owner: true" } ] diff --git a/metadata-ingestion/docs/sources/qlik-cloud/qlik-cloud_pre.md b/metadata-ingestion/docs/sources/qlik-cloud/qlik-cloud_pre.md deleted file mode 100644 index 328a50c9462c3..0000000000000 --- a/metadata-ingestion/docs/sources/qlik-cloud/qlik-cloud_pre.md +++ /dev/null @@ -1,19 +0,0 @@ -## Integration Details - -This source extracts the following: - -- Accessible spaces and apps within that spaces as Container. -- Qlik Datasets as Datahub Datasets with schema metadata. - -## Configuration Notes - -1. Refer [doc](https://qlik.dev/authenticate/api-key/generate-your-first-api-key/) to generate an API key from the hub. -2. Get tenant hostname from About tab after login to qlik sense account. - -## Concept mapping - -| Qlik Cloud | Datahub | -|--------------------------|--------------------------------------------------------------------------------------------------------| -| `Space` | [Container](https://datahubproject.io/docs/generated/metamodel/entities/container) | -| `App` | [Container](https://datahubproject.io/docs/generated/metamodel/entities/container) | -| `Dataset` | [Dataset](https://datahubproject.io/docs/generated/metamodel/entities/dataset) | \ No newline at end of file diff --git a/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_pre.md b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_pre.md new file mode 100644 index 0000000000000..c344a32d07676 --- /dev/null +++ b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_pre.md @@ -0,0 +1,23 @@ +## Integration Details + +This source extracts the following: + +- Accessible spaces and apps within that spaces as Container. +- Qlik Datasets as Datahub Datasets with schema metadata. +- Sheets as Datahub dashboard and charts present inside sheets. + +## Configuration Notes + +1. Refer [doc](https://qlik.dev/authenticate/api-key/generate-your-first-api-key/) to generate an API key from the hub. +2. Get tenant hostname from About tab after login to qlik sense account. + +## Concept mapping + +| Qlik Sense | Datahub | Notes | +|------------------------|---------------------------------------------------------------|----------------------------------| +| `Space` | [Container](../../metamodel/entities/container.md) | SubType `"Qlik Space"` | +| `App` | [Container](../../metamodel/entities/container.md) | SubType `"Qlik App"` | +| `Sheet` | [Dashboard](../../metamodel/entities/dashboard.md) | | +| `Chart` | [Chart](../../metamodel/entities/chart.md) | | +| `Dataset` | [Dataset](../../metamodel/entities/dataset.md) | SubType `"Qlik Dataset"` | +| `User` | [User (a.k.a CorpUser)](../../metamodel/entities/corpuser.md) | Optionally Extracted | \ No newline at end of file diff --git a/metadata-ingestion/docs/sources/qlik-cloud/qlik-cloud_recipe.yml b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml similarity index 95% rename from metadata-ingestion/docs/sources/qlik-cloud/qlik-cloud_recipe.yml rename to metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml index 5d2e20b8b039c..97ab4750b9d09 100644 --- a/metadata-ingestion/docs/sources/qlik-cloud/qlik-cloud_recipe.yml +++ b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml @@ -1,5 +1,5 @@ source: - type: qlik-cloud + type: qlik-sense config: # Coordinates tenant_hostname: "https://xyz12xz.us.qlikcloud.com" diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 203da0f053bee..f4f2a2320275e 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -397,7 +397,7 @@ # databricks is alias for unity-catalog and needs to be kept in sync "databricks": databricks | sql_common | sqllineage_lib, "fivetran": snowflake_common | bigquery_common, - "qlik-cloud": {"requests"}, + "qlik-sense": {"requests"}, } # This is mainly used to exclude plugins from the Docker image. @@ -627,7 +627,7 @@ "gcs = datahub.ingestion.source.gcs.gcs_source:GCSSource", "sql-queries = datahub.ingestion.source.sql_queries:SqlQueriesSource", "fivetran = datahub.ingestion.source.fivetran.fivetran:FivetranSource", - "qlik-cloud = datahub.ingestion.source.qlik_cloud.qlik_cloud:QlikCloudSource", + "qlik-sense = datahub.ingestion.source.qlik_sense.qlik_sense:QlikSenseSource", ], "datahub.ingestion.transformer.plugins": [ "simple_remove_dataset_ownership = datahub.ingestion.transformer.remove_dataset_ownership:SimpleRemoveDatasetOwnership", diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/data_classes.py deleted file mode 100644 index e740f88816687..0000000000000 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/data_classes.py +++ /dev/null @@ -1,128 +0,0 @@ -from dataclasses import dataclass -from datetime import datetime -from enum import Enum -from typing import Dict, List, Optional, Type, Union - -from datahub.emitter.mcp_builder import ContainerKey -from datahub.metadata.com.linkedin.pegasus2avro.schema import ( - BooleanType, - BytesType, - DateType, - NumberType, - StringType, - TimeType, -) - -FIELD_TYPE_MAPPING: Dict[ - str, - Type[ - Union[ - BooleanType, - BytesType, - DateType, - NumberType, - StringType, - TimeType, - ] - ], -] = { - "DATE": DateType, - "TIME": TimeType, - "DATETIME": DateType, - "TIMESTAMP": DateType, - "STRING": StringType, - "DOUBLE": NumberType, - "DECIMAL": NumberType, - "INTEGER": NumberType, - "BOOLEAN": BooleanType, - "BINARY": BytesType, -} - - -class SpaceKey(ContainerKey): - space: str - - -class AppKey(ContainerKey): - app: str - - -class SpaceType(Enum): - PERSONAL = "personal" - SHARED = "shared" - MANAGED = "managed" - DATA = "data" - - -@dataclass -class Space: - id: str - name: str - description: str - type: SpaceType - created_at: datetime - updated_at: datetime - owner_id: Optional[str] = None - - -@dataclass -class SchemaField: - name: str - data_type: str - primary_key: bool - nullable: bool - - -@dataclass -class Item: - id: str - name: str - qri: str - description: str - owner_id: str - space_id: str - created_at: datetime - updated_at: datetime - - -@dataclass -class App(Item): - usage: str - - -@dataclass -class QlikDataset(Item): - type: str - size: int - row_count: int - schema: List[SchemaField] - - -@dataclass -class User: - id: str - displayName: str - emailAddress: str - graphId: str - principalType: str - datasetUserAccessRight: Optional[str] = None - reportUserAccessRight: Optional[str] = None - dashboardUserAccessRight: Optional[str] = None - groupUserAccessRight: Optional[str] = None - - def get_urn_part(self, use_email: bool, remove_email_suffix: bool) -> str: - if use_email: - if remove_email_suffix: - return self.emailAddress.split("@")[0] - else: - return self.emailAddress - return f"users.{self.id}" - - def __members(self): - return (self.id,) - - def __eq__(self, instance): - return isinstance(instance, User) and self.__members() == instance.__members() - - def __hash__(self): - return hash(self.__members()) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py deleted file mode 100644 index 3f66aba322db3..0000000000000 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_api.py +++ /dev/null @@ -1,144 +0,0 @@ -import logging -import sys -from datetime import datetime -from typing import Any, List, Optional - -import requests - -from datahub.ingestion.source.qlik_cloud.config import ( - QLIK_DATETIME_FORMAT, - Constant, - QlikSourceConfig, -) -from datahub.ingestion.source.qlik_cloud.data_classes import ( - App, - Item, - QlikDataset, - SchemaField, - Space, -) - -# Logger instance -logger = logging.getLogger(__name__) - - -class QlikAPI: - def __init__(self, config: QlikSourceConfig) -> None: - self.config = config - self.session = requests.Session() - self.session.headers.update( - { - "Authorization": f"Bearer {self.config.api_key}", - "Content-Type": "application/json", - } - ) - self.base_url = f"{self.config.tenant_hostname}/api/v1" - # Test connection by fetching list of api keys - logger.info("Trying to connect to {}".format(self.base_url)) - self.session.get(f"{self.base_url}/api-keys").raise_for_status() - - def log_http_error(self, message: str) -> Any: - logger.warning(message) - _, e, _ = sys.exc_info() - if isinstance(e, requests.exceptions.HTTPError): - logger.warning(f"HTTP status-code = {e.response.status_code}") - logger.debug(msg=message, exc_info=e) - return e - - def get_spaces(self) -> List[Space]: - spaces: List[Space] = [] - try: - response = self.session.get(f"{self.base_url}/spaces") - response.raise_for_status() - for space in response.json()[Constant.DATA]: - spaces.append( - Space( - id=space[Constant.ID], - name=space[Constant.NAME], - description=space[Constant.DESCRIPTION], - type=space[Constant.TYPE], - owner_id=space[Constant.OWNERID], - created_at=datetime.strptime( - space[Constant.CREATEDAT], QLIK_DATETIME_FORMAT - ), - updated_at=datetime.strptime( - space[Constant.UPDATEDAT], QLIK_DATETIME_FORMAT - ), - ) - ) - except Exception: - self.log_http_error(message="Unable to fetch spaces") - return spaces - - def _get_dataset(self, dataset_id: str) -> Optional[QlikDataset]: - try: - response = self.session.get(f"{self.base_url}/data-sets/{dataset_id}") - response.raise_for_status() - dataset = response.json() - return QlikDataset( - id=dataset[Constant.ID], - name=dataset[Constant.NAME], - qri=dataset[Constant.SECUREQRI], - description=dataset[Constant.DESCRIPTION], - space_id=dataset.get(Constant.SPACEID, ""), - owner_id=dataset[Constant.OWNERID], - created_at=datetime.strptime( - dataset[Constant.CREATEDTIME], QLIK_DATETIME_FORMAT - ), - updated_at=datetime.strptime( - dataset[Constant.LASTMODIFIEDTIME], QLIK_DATETIME_FORMAT - ), - type=dataset[Constant.TYPE], - size=dataset[Constant.OPERATIONAL].get(Constant.SIZE, 0), - row_count=dataset[Constant.OPERATIONAL][Constant.ROWCOUNT], - schema=[ - SchemaField( - name=field[Constant.NAME], - data_type=field[Constant.DATATYPE][Constant.TYPE], - primary_key=field[Constant.PRIMARYKEY], - nullable=field[Constant.NULLABLE], - ) - for field in dataset[Constant.SCHEMA][Constant.DATAFIELDS] - ], - ) - except Exception: - self.log_http_error(message="Unable to fetch items") - return None - - def get_items(self) -> List[Item]: - items: List[Item] = [] - try: - response = self.session.get(f"{self.base_url}/items") - response.raise_for_status() - data = response.json()[Constant.DATA] - for item in data: - resource_type = item[Constant.RESOURCETYPE] - resource_attributes = item[Constant.RESOURCEATTRIBUTES] - if resource_type == Constant.APP: - items.append( - App( - id=resource_attributes[Constant.ID], - name=resource_attributes[Constant.NAME], - qri=f"qri:app:sense://{resource_attributes[Constant.ID]}", - description=resource_attributes[Constant.DESCRIPTION], - space_id=resource_attributes[Constant.SPACEID], - usage=resource_attributes[Constant.USAGE], - owner_id=resource_attributes[Constant.OWNERID], - created_at=datetime.strptime( - resource_attributes[Constant.CREATEDDATE], - QLIK_DATETIME_FORMAT, - ), - updated_at=datetime.strptime( - resource_attributes[Constant.MODIFIEDDATE], - QLIK_DATETIME_FORMAT, - ), - ) - ) - elif resource_type == Constant.DATASET: - dataset = self._get_dataset(dataset_id=item[Constant.RESOURCEID]) - if dataset: - items.append(dataset) - - except Exception: - self.log_http_error(message="Unable to fetch items") - return items diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/__init__.py similarity index 100% rename from metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/__init__.py rename to metadata-ingestion/src/datahub/ingestion/source/qlik_sense/__init__.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/config.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py similarity index 73% rename from metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/config.py rename to metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py index 17644d34b46d4..02ee092f7709e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py @@ -5,10 +5,12 @@ import pydantic from datahub.configuration.common import AllowDenyPattern -from datahub.configuration.source_common import DatasetSourceConfigMixin +from datahub.configuration.source_common import ( + EnvConfigMixin, + PlatformInstanceConfigMixin, +) from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalSourceReport, - StatefulStaleMetadataRemovalConfig, ) from datahub.ingestion.source.state.stateful_ingestion_base import ( StatefulIngestionConfigBase, @@ -19,18 +21,29 @@ QLIK_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" +class WebSocketRequest: + """ + Web socket send request dict + """ + + OPEN_DOC = { + "jsonrpc": "2.0", + "id": 1, + "handle": -1, + "method": "OpenDoc", + "params": {"qDocName": "f0714ca7-7093-49e4-8b58-47bb38563647"}, + } + + class Constant: """ keys used in qlik plugin """ - PLATFORM_NAME = "qlik-cloud" + # Rest API response key constants DATA = "data" - ID = "id" NAME = "name" - DESCRIPTION = "description" TYPE = "type" - OWNERID = "ownerId" CREATEDAT = "createdAt" UPDATEDAT = "updatedAt" SECUREQRI = "secureQri" @@ -52,6 +65,18 @@ class Constant: CREATEDDATE = "createdDate" MODIFIEDDATE = "modifiedDate" RESOURCEID = "resourceId" + DATASETSCHEMA = "datasetSchema" + # Websocket response key constants + QID = "qId" + RESULT = "result" + QRETURN = "qReturn" + QHANDLE = "qHandle" + QLAYOUT = "qLayout" + QMETA = "qMeta" + QCHILDLIST = "qChildList" + QITEMS = "qItems" + QINFO = "qInfo" + QLIST = "qList" # Item type APP = "app" DATASET = "dataset" @@ -68,7 +93,9 @@ def report_number_of_spaces(self, number_of_spaces: int) -> None: self.number_of_spaces = number_of_spaces -class QlikSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin): +class QlikSourceConfig( + StatefulIngestionConfigBase, PlatformInstanceConfigMixin, EnvConfigMixin +): tenant_hostname: str = pydantic.Field(description="Qlik Tenant hostname") api_key: str = pydantic.Field(description="Qlik API Key") # Qlik space identifier @@ -84,7 +111,3 @@ class QlikSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin): default=True, description="Ingest Owner from source. This will override Owner info entered from UI", ) - # Configuration for stateful ingestion - stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = pydantic.Field( - default=None, description="Qlik Stateful Ingestion Config." - ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py new file mode 100644 index 0000000000000..17219636f0173 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py @@ -0,0 +1,182 @@ +from datetime import datetime +from enum import Enum +from typing import Dict, List, Optional, Type, Union + +from pydantic import BaseModel, root_validator + +from datahub.emitter.mcp_builder import ContainerKey +from datahub.ingestion.source.qlik_sense.config import QLIK_DATETIME_FORMAT, Constant +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + BooleanType, + BytesType, + DateType, + NumberType, + StringType, + TimeType, +) + +FIELD_TYPE_MAPPING: Dict[ + str, + Type[ + Union[ + BooleanType, + BytesType, + DateType, + NumberType, + StringType, + TimeType, + ] + ], +] = { + "DATE": DateType, + "TIME": TimeType, + "DATETIME": DateType, + "TIMESTAMP": DateType, + "STRING": StringType, + "DOUBLE": NumberType, + "DECIMAL": NumberType, + "INTEGER": NumberType, + "BOOLEAN": BooleanType, + "BINARY": BytesType, +} + + +class SpaceKey(ContainerKey): + space: str + + +class AppKey(ContainerKey): + app: str + + +class SpaceType(Enum): + PERSONAL = "personal" + SHARED = "shared" + MANAGED = "managed" + DATA = "data" + + +PERSONAL_SPACE_DICT = { + "id": Constant.PERSONAL_SPACE_ID, + "name": Constant.PERSONAL_SPACE_NAME, + "description": "", + "type": SpaceType.PERSONAL, + "createdAt": datetime.now().strftime(QLIK_DATETIME_FORMAT), + "updatedAt": datetime.now().strftime(QLIK_DATETIME_FORMAT), +} + + +class Space(BaseModel): + id: str + name: str + description: str + type: SpaceType + createdAt: datetime + updatedAt: datetime + ownerId: Optional[str] = None + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.CREATEDAT] = datetime.strptime( + values[Constant.CREATEDAT], QLIK_DATETIME_FORMAT + ) + values[Constant.UPDATEDAT] = datetime.strptime( + values[Constant.UPDATEDAT], QLIK_DATETIME_FORMAT + ) + return values + + +class Item(BaseModel): + id: str + name: str + qri: str + description: str + ownerId: str + spaceId: str + createdAt: datetime + updatedAt: datetime + + +class Chart(BaseModel): + qId: str + qType: str + + +class Sheet(BaseModel): + id: str + title: str + description: str + ownerId: str + createdAt: datetime + updatedAt: datetime + charts: List[Chart] = [] + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.CREATEDAT] = datetime.strptime( + values[Constant.CREATEDDATE], QLIK_DATETIME_FORMAT + ) + values[Constant.UPDATEDAT] = datetime.strptime( + values[Constant.MODIFIEDDATE], QLIK_DATETIME_FORMAT + ) + return values + + +class App(Item): + usage: str + sheets: List[Sheet] = [] + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.CREATEDAT] = datetime.strptime( + values[Constant.CREATEDDATE], QLIK_DATETIME_FORMAT + ) + values[Constant.UPDATEDAT] = datetime.strptime( + values[Constant.MODIFIEDDATE], QLIK_DATETIME_FORMAT + ) + if not values.get(Constant.SPACEID): + # spaceId none indicates app present in personal space + values[Constant.SPACEID] = Constant.PERSONAL_SPACE_ID + values[Constant.QRI] = f"qri:app:sense://{values['id']}" + return values + + +class SchemaField(BaseModel): + name: str + dataType: str + primaryKey: bool + nullable: bool + + +class QlikDataset(Item): + type: str + size: int + rowCount: int + datasetSchema: List[SchemaField] + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + # Update str time to datetime + values[Constant.CREATEDAT] = datetime.strptime( + values[Constant.CREATEDTIME], QLIK_DATETIME_FORMAT + ) + values[Constant.UPDATEDAT] = datetime.strptime( + values[Constant.LASTMODIFIEDTIME], QLIK_DATETIME_FORMAT + ) + if not values.get(Constant.SPACEID): + # spaceId none indicates dataset present in personal space + values[Constant.SPACEID] = Constant.PERSONAL_SPACE_ID + values[Constant.QRI] = values[Constant.SECUREQRI] + values[Constant.SIZE] = values[Constant.OPERATIONAL].get(Constant.SIZE, 0) + values[Constant.ROWCOUNT] = values[Constant.OPERATIONAL][Constant.ROWCOUNT] + + values[Constant.DATASETSCHEMA] = [ + { + Constant.NAME: field[Constant.NAME], + Constant.DATATYPE: field[Constant.DATATYPE][Constant.TYPE], + Constant.PRIMARYKEY: field[Constant.PRIMARYKEY], + Constant.NULLABLE: field[Constant.NULLABLE], + } + for field in values[Constant.SCHEMA][Constant.DATAFIELDS] + ] + return values diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py new file mode 100644 index 0000000000000..6abc4c931cf9c --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py @@ -0,0 +1,196 @@ +import json +import logging +import sys +from typing import Any, Dict, List, Optional + +import requests +from websocket import WebSocket, create_connection + +from datahub.ingestion.source.qlik_sense.config import Constant, QlikSourceConfig +from datahub.ingestion.source.qlik_sense.data_classes import ( + App, + Chart, + Item, + QlikDataset, + Sheet, + Space, +) + +# Logger instance +logger = logging.getLogger(__name__) + + +class QlikAPI: + def __init__(self, config: QlikSourceConfig) -> None: + self.config = config + self.session = requests.Session() + self.session.headers.update( + { + "Authorization": f"Bearer {self.config.api_key}", + "Content-Type": "application/json", + } + ) + self.rest_api_url = f"https://{self.config.tenant_hostname}/api/v1" + self.websocket_url = f"wss://{self.config.tenant_hostname}/app" + # Test connection by fetching list of api keys + logger.info("Trying to connect to {}".format(self.rest_api_url)) + self.session.get(f"{self.rest_api_url}/api-keys").raise_for_status() + + def _log_http_error(self, message: str) -> Any: + logger.warning(message) + _, e, _ = sys.exc_info() + if isinstance(e, requests.exceptions.HTTPError): + logger.warning(f"HTTP status-code = {e.response.status_code}") + logger.debug(msg=message, exc_info=e) + return e + + def _websocket_send_request( + self, socket_connection: WebSocket, request: dict + ) -> Dict: + """ + Method to send request to websocket + """ + socket_connection.send(json.dumps(request)) + resp = socket_connection.recv() + if request["handle"] == -1: + resp = socket_connection.recv() + return json.loads(resp) + + def _get_websocket_request_dict( + self, id: int, handle: int, method: str, params: Dict = {} + ) -> Dict: + return { + "jsonrpc": "2.0", + "id": id, + "handle": handle, + "method": method, + "params": params, + } + + def get_spaces(self) -> List[Space]: + spaces: List[Space] = [] + try: + response = self.session.get(f"{self.rest_api_url}/spaces") + response.raise_for_status() + for space_dict in response.json()[Constant.DATA]: + spaces.append(Space.parse_obj(space_dict)) + except Exception: + self._log_http_error(message="Unable to fetch spaces") + return spaces + + def _get_dataset(self, dataset_id: str) -> Optional[QlikDataset]: + try: + response = self.session.get(f"{self.rest_api_url}/data-sets/{dataset_id}") + response.raise_for_status() + return QlikDataset.parse_obj(response.json()) + except Exception: + self._log_http_error( + message=f"Unable to fetch dataset with id {dataset_id}" + ) + return None + + def get_items(self) -> List[Item]: + items: List[Item] = [] + try: + response = self.session.get(f"{self.rest_api_url}/items") + response.raise_for_status() + data = response.json()[Constant.DATA] + for item in data: + resource_type = item[Constant.RESOURCETYPE] + resource_attributes = item[Constant.RESOURCEATTRIBUTES] + if resource_type == Constant.APP: + app = App.parse_obj(resource_attributes) + app.sheets = self._get_app_sheets(app_id=app.id) + items.append(app) + elif resource_type == Constant.DATASET: + dataset = self._get_dataset(dataset_id=item[Constant.RESOURCEID]) + if dataset: + items.append(dataset) + + except Exception: + self._log_http_error(message="Unable to fetch items") + return items + + def _get_sheet( + self, + socket_connection: WebSocket, + request_id: int, + current_handle: int, + sheet_id: str, + ) -> Optional[Sheet]: + try: + response = self._websocket_send_request( + socket_connection=socket_connection, + request=self._get_websocket_request_dict( + id=request_id, + handle=current_handle, + method="GetObject", + params={"qId": sheet_id}, + ), + ) + request_id += 1 + current_handle = response[Constant.RESULT][Constant.QRETURN][ + Constant.QHANDLE + ] + response = self._websocket_send_request( + socket_connection=socket_connection, + request=self._get_websocket_request_dict( + id=request_id, handle=current_handle, method="GetLayout" + ), + ) + request_id += 1 + sheet_dict = response[Constant.RESULT][Constant.QLAYOUT] + sheet = Sheet.parse_obj(sheet_dict[Constant.QMETA]) + for chart_dict in sheet_dict[Constant.QCHILDLIST][Constant.QITEMS]: + sheet.charts.append(Chart.parse_obj(chart_dict[Constant.QINFO])) + return sheet + except Exception: + self._log_http_error(message=f"Unable to fetch sheet with id {sheet_id}") + return None + + def _get_app_sheets(self, app_id: str) -> List[Sheet]: + sheets: List[Sheet] = [] + request_id = 1 + current_handle = -1 + try: + socket_connection = create_connection( + f"{self.websocket_url}/{app_id}", + header={"Authorization": f"Bearer {self.config.api_key}"}, + ) + response = self._websocket_send_request( + socket_connection=socket_connection, + request=self._get_websocket_request_dict( + id=request_id, + handle=current_handle, + method="OpenDoc", + params={"qDocName": app_id}, + ), + ) + request_id += 1 + current_handle = response["result"]["qReturn"]["qHandle"] + response = self._websocket_send_request( + socket_connection=socket_connection, + request=self._get_websocket_request_dict( + id=request_id, + handle=current_handle, + method="GetObjects", + params={ + "qOptions": { + "qTypes": ["sheet"], + } + }, + ), + ) + request_id += 1 + for sheet_dict in response[Constant.RESULT][Constant.QLIST]: + sheet = self._get_sheet( + socket_connection=socket_connection, + request_id=request_id, + current_handle=current_handle, + sheet_id=sheet_dict[Constant.QINFO][Constant.QID], + ) + if sheet: + sheets.append(sheet) + except Exception: + self._log_http_error(message="Unable to fetch sheets") + return sheets diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py similarity index 55% rename from metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py rename to metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py index a7e5fe0fbbcc8..535b4f5b7a565 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_cloud/qlik_cloud.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py @@ -1,15 +1,10 @@ import logging from datetime import datetime -from typing import Iterable, List, Optional +from typing import Dict, Iterable, List, Optional -from datahub.emitter.mce_builder import ( - make_data_platform_urn, - make_dataplatform_instance_urn, - make_dataset_urn_with_platform_instance, - make_user_urn, -) +import datahub.emitter.mce_builder as builder from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.emitter.mcp_builder import add_dataset_to_container, gen_containers +from datahub.emitter.mcp_builder import add_entity_to_container, gen_containers from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SourceCapability, @@ -31,22 +26,24 @@ BIContainerSubTypes, DatasetSubTypes, ) -from datahub.ingestion.source.qlik_cloud.config import ( +from datahub.ingestion.source.qlik_sense.config import ( Constant, QlikSourceConfig, QlikSourceReport, ) -from datahub.ingestion.source.qlik_cloud.data_classes import ( +from datahub.ingestion.source.qlik_sense.data_classes import ( FIELD_TYPE_MAPPING, + PERSONAL_SPACE_DICT, App, AppKey, + Chart, QlikDataset, SchemaField as QlikDatasetSchemaField, + Sheet, Space, SpaceKey, - SpaceType, ) -from datahub.ingestion.source.qlik_cloud.qlik_api import QlikAPI +from datahub.ingestion.source.qlik_sense.qlik_api import QlikAPI from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalHandler, ) @@ -66,6 +63,10 @@ SchemaMetadata, ) from datahub.metadata.schema_classes import ( + AuditStampClass, + ChangeAuditStampsClass, + ChartInfoClass, + DashboardInfoClass, DataPlatformInstanceClass, OwnerClass, OwnershipClass, @@ -77,7 +78,7 @@ logger = logging.getLogger(__name__) -@platform_name("Qlik Cloud") +@platform_name("Qlik Sense") @config_class(QlikSourceConfig) @support_status(SupportStatus.INCUBATING) @capability(SourceCapability.DESCRIPTIONS, "Enabled by default") @@ -86,17 +87,20 @@ SourceCapability.OWNERSHIP, "Enabled by default, configured using `ingest_owner`", ) -class QlikCloudSource(StatefulIngestionSourceBase, TestableSource): +class QlikSenseSource(StatefulIngestionSourceBase, TestableSource): """ - This plugin extracts Qlik Cloud Spaces, Apps, and Datasets + This plugin extracts the following: + - Qlik Sense Spaces and Apps as Container. + - Qlik Datasets + - Sheets as dashboard and its charts """ config: QlikSourceConfig reporter: QlikSourceReport - platform: str = "qlik-cloud" + platform: str = "qlik-sense" def __init__(self, config: QlikSourceConfig, ctx: PipelineContext): - super(QlikCloudSource, self).__init__(config, ctx) + super(QlikSenseSource, self).__init__(config, ctx) self.config = config self.reporter = QlikSourceReport() try: @@ -139,13 +143,12 @@ def _gen_app_key(self, app_id: str) -> AppKey: ) def _gen_personal_space(self) -> Space: - return Space( - id=Constant.PERSONAL_SPACE_ID, - name=Constant.PERSONAL_SPACE_NAME, - description="", - type=SpaceType.PERSONAL, - created_at=datetime.now(), - updated_at=datetime.now(), + return Space.parse_obj(PERSONAL_SPACE_DICT) + + def _get_audit_stamp(self, date: datetime, username: str) -> AuditStampClass: + return AuditStampClass( + time=int(date.timestamp() * 1000), + actor=builder.make_user_urn(username), ) def _get_allowed_spaces(self) -> List[Space]: @@ -164,71 +167,178 @@ def _get_allowed_spaces(self) -> List[Space]: return allowed_spaces def _gen_space_workunit(self, space: Space) -> Iterable[MetadataWorkUnit]: + """ + Map Qlik space to Datahub container + """ yield from gen_containers( container_key=self._gen_space_key(space.id), name=space.name, description=space.description, sub_types=[BIContainerSubTypes.QLIK_SPACE], extra_properties={Constant.TYPE: str(space.type)}, - owner_urn=make_user_urn(space.owner_id) - if self.config.ingest_owner and space.owner_id + owner_urn=builder.make_user_urn(space.ownerId) + if self.config.ingest_owner and space.ownerId else None, - created=int(space.created_at.timestamp() * 1000), - last_modified=int(space.updated_at.timestamp() * 1000), + created=int(space.createdAt.timestamp() * 1000), + last_modified=int(space.updatedAt.timestamp() * 1000), + ) + + def _gen_entity_status_aspect(self, entity_urn: str) -> MetadataWorkUnit: + return MetadataChangeProposalWrapper( + entityUrn=entity_urn, aspect=Status(removed=False) + ).as_workunit() + + def _gen_entity_owner_aspect( + self, entity_urn: str, owner_id: str + ) -> MetadataWorkUnit: + aspect = OwnershipClass( + owners=[ + OwnerClass( + owner=builder.make_user_urn(owner_id), + type=OwnershipTypeClass.DATAOWNER, + ) + ] + ) + return MetadataChangeProposalWrapper( + entityUrn=entity_urn, + aspect=aspect, + ).as_workunit() + + def _gen_dashboard_urn(self, dashboard_identifier: str) -> str: + return builder.make_dashboard_urn( + platform=self.platform, + platform_instance=self.config.platform_instance, + name=dashboard_identifier, + ) + + def _gen_dashboard_info_workunit(self, sheet: Sheet) -> MetadataWorkUnit: + dashboard_urn = self._gen_dashboard_urn(sheet.id) + custom_properties: Dict[str, str] = {"chartCount": str(len(sheet.charts))} + dashboard_info_cls = DashboardInfoClass( + title=sheet.title, + description=sheet.description, + charts=[ + builder.make_chart_urn( + platform=self.platform, + platform_instance=self.config.platform_instance, + name=chart.qId, + ) + for chart in sheet.charts + ], + lastModified=ChangeAuditStampsClass( + created=self._get_audit_stamp(sheet.createdAt, sheet.ownerId), + lastModified=self._get_audit_stamp(sheet.updatedAt, sheet.ownerId), + ), + customProperties=custom_properties, ) + return MetadataChangeProposalWrapper( + entityUrn=dashboard_urn, aspect=dashboard_info_cls + ).as_workunit() + + def _gen_charts_workunit( + self, charts: List[Chart], space_id: str, app_id: str, sheet_id: str + ) -> Iterable[MetadataWorkUnit]: + """ + Map Qlik Chart to Datahub Chart + """ + for chart in charts: + chart_urn = builder.make_chart_urn( + platform=self.platform, + platform_instance=self.config.platform_instance, + name=chart.qId, + ) + + yield self._gen_entity_status_aspect(chart_urn) + + yield MetadataChangeProposalWrapper( + entityUrn=chart_urn, + aspect=ChartInfoClass( + title=chart.qId, + description=chart.qType, + lastModified=ChangeAuditStampsClass(), + ), + ).as_workunit() + + # yield MetadataChangeProposalWrapper( + # entityUrn=chart_urn, + # aspect=BrowsePathsClass( + # paths=[f"/{self.platform}/{space_id}/{app_id}/{sheet_id}"] + # ), + # ).as_workunit() + + def _gen_sheets_workunit( + self, sheets: List[Sheet], space_id: str, app_id: str + ) -> Iterable[MetadataWorkUnit]: + """ + Map Qlik Sheet to Datahub dashboard + """ + for sheet in sheets: + dashboard_urn = self._gen_dashboard_urn(sheet.id) + + yield self._gen_entity_status_aspect(dashboard_urn) + + yield self._gen_dashboard_info_workunit(sheet) + + yield from add_entity_to_container( + container_key=self._gen_app_key(app_id), + entity_type="dashboard", + entity_urn=dashboard_urn, + ) + + dpi_aspect = self._gen_dataplatform_instance_aspect(dashboard_urn) + if dpi_aspect: + yield dpi_aspect + + if self.config.ingest_owner: + yield self._gen_entity_owner_aspect(dashboard_urn, sheet.ownerId) + + yield from self._gen_charts_workunit( + sheet.charts, space_id, app_id, sheet.id + ) def _gen_app_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: + """ + Map Qlik App to Datahub container + """ yield from gen_containers( container_key=self._gen_app_key(app.id), name=app.name, description=app.description, sub_types=[BIContainerSubTypes.QLIK_APP], - parent_container_key=self._gen_space_key(app.space_id), + parent_container_key=self._gen_space_key(app.spaceId), extra_properties={Constant.QRI: app.qri, Constant.USAGE: app.usage}, - owner_urn=make_user_urn(app.owner_id) if self.config.ingest_owner else None, - created=int(app.created_at.timestamp() * 1000), - last_modified=int(app.updated_at.timestamp() * 1000), + owner_urn=builder.make_user_urn(app.ownerId) + if self.config.ingest_owner + else None, + created=int(app.createdAt.timestamp() * 1000), + last_modified=int(app.updatedAt.timestamp() * 1000), ) + yield from self._gen_sheets_workunit(app.sheets, app.spaceId, app.id) def _gen_dataset_urn(self, dataset_identifier: str) -> str: - return make_dataset_urn_with_platform_instance( + return builder.make_dataset_urn_with_platform_instance( name=dataset_identifier, env=self.config.env, platform=self.platform, platform_instance=self.config.platform_instance, ) - def gen_dataplatform_instance_aspect( - self, dataset_urn: str + def _gen_dataplatform_instance_aspect( + self, entity_urn: str ) -> Optional[MetadataWorkUnit]: if self.config.platform_instance: aspect = DataPlatformInstanceClass( - platform=make_data_platform_urn(self.platform), - instance=make_dataplatform_instance_urn( + platform=builder.make_data_platform_urn(self.platform), + instance=builder.make_dataplatform_instance_urn( self.platform, self.config.platform_instance ), ) return MetadataChangeProposalWrapper( - entityUrn=dataset_urn, aspect=aspect + entityUrn=entity_urn, aspect=aspect ).as_workunit() else: return None - def _gen_dataset_owner_aspect( - self, dataset_urn: str, owner_id: str - ) -> MetadataWorkUnit: - aspect = OwnershipClass( - owners=[ - OwnerClass( - owner=make_user_urn(owner_id), type=OwnershipTypeClass.DATAOWNER - ) - ] - ) - return MetadataChangeProposalWrapper( - entityUrn=dataset_urn, - aspect=aspect, - ).as_workunit() - def _gen_schema_fields( self, schema: List[QlikDatasetSchemaField] ) -> List[SchemaField]: @@ -237,12 +347,12 @@ def _gen_schema_fields( schema_field = SchemaField( fieldPath=field.name, type=SchemaFieldDataTypeClass( - type=FIELD_TYPE_MAPPING.get(field.data_type, NullType)() + type=FIELD_TYPE_MAPPING.get(field.dataType, NullType)() ), # NOTE: nativeDataType will not be in sync with older connector - nativeDataType=field.data_type, + nativeDataType=field.dataType, nullable=field.nullable, - isPartOfKey=field.primary_key, + isPartOfKey=field.primaryKey, ) schema_fields.append(schema_field) return schema_fields @@ -252,11 +362,11 @@ def _gen_schema_metadata(self, dataset: QlikDataset) -> MetadataWorkUnit: schema_metadata = SchemaMetadata( schemaName=dataset.id, - platform=make_data_platform_urn(self.platform), + platform=builder.make_data_platform_urn(self.platform), version=0, hash="", platformSchema=MySqlDDL(tableSchema=""), - fields=self._gen_schema_fields(dataset.schema), + fields=self._gen_schema_fields(dataset.datasetSchema), ) return MetadataChangeProposalWrapper( @@ -270,16 +380,16 @@ def _gen_dataset_properties(self, dataset: QlikDataset) -> MetadataWorkUnit: name=dataset.name, description=dataset.description, qualifiedName=dataset.name, - created=TimeStamp(time=int(dataset.created_at.timestamp() * 1000)), - lastModified=TimeStamp(time=int(dataset.updated_at.timestamp() * 1000)), + created=TimeStamp(time=int(dataset.createdAt.timestamp() * 1000)), + lastModified=TimeStamp(time=int(dataset.updatedAt.timestamp() * 1000)), ) dataset_properties.customProperties.update( { Constant.QRI: dataset.qri, - Constant.SPACEID: dataset.space_id, + Constant.SPACEID: dataset.spaceId, Constant.TYPE: dataset.type, Constant.SIZE: str(dataset.size), - Constant.ROWCOUNT: str(dataset.row_count), + Constant.ROWCOUNT: str(dataset.rowCount), } ) @@ -290,25 +400,24 @@ def _gen_dataset_properties(self, dataset: QlikDataset) -> MetadataWorkUnit: def _gen_dataset_workunit(self, dataset: QlikDataset) -> Iterable[MetadataWorkUnit]: dataset_urn = self._gen_dataset_urn(dataset.id) - yield MetadataChangeProposalWrapper( - entityUrn=dataset_urn, aspect=Status(removed=False) - ).as_workunit() + yield self._gen_entity_status_aspect(dataset_urn) yield self._gen_schema_metadata(dataset) yield self._gen_dataset_properties(dataset) - yield from add_dataset_to_container( - container_key=self._gen_space_key(dataset.space_id), - dataset_urn=dataset_urn, + yield from add_entity_to_container( + container_key=self._gen_space_key(dataset.spaceId), + entity_type="dataset", + entity_urn=dataset_urn, ) - dpi_aspect = self.gen_dataplatform_instance_aspect(dataset_urn) + dpi_aspect = self._gen_dataplatform_instance_aspect(dataset_urn) if dpi_aspect: yield dpi_aspect if self.config.ingest_owner: - yield self._gen_dataset_owner_aspect(dataset_urn, dataset.owner_id) + yield self._gen_entity_owner_aspect(dataset_urn, dataset.ownerId) yield MetadataChangeProposalWrapper( entityUrn=dataset_urn, @@ -327,15 +436,16 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: """ Datahub Ingestion framework invoke this method """ - logger.info("Qlik Cloud plugin execution is started") + logger.info("Qlik Sense plugin execution is started") for space in self._get_allowed_spaces(): yield from self._gen_space_workunit(space) for item in self.qlik_api.get_items(): # If item is personal item and flag is Disable, skip ingesting personal item - if not item.space_id: - item.space_id = Constant.PERSONAL_SPACE_ID - if not self.config.extract_personal_entity: - continue + if ( + item.spaceId == Constant.PERSONAL_SPACE_ID + and not self.config.extract_personal_entity + ): + continue if isinstance(item, App): yield from self._gen_app_workunit(item) elif isinstance(item, QlikDataset): diff --git a/metadata-ingestion/tests/integration/qlik_cloud/golden_test_platform_instace_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instace_ingest.json similarity index 54% rename from metadata-ingestion/tests/integration/qlik_cloud/golden_test_platform_instace_ingest.json rename to metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instace_ingest.json index b84bf67c8178e..f991105f0a5a1 100644 --- a/metadata-ingestion/tests/integration/qlik_cloud/golden_test_platform_instace_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instace_ingest.json @@ -1,16 +1,16 @@ [ { "entityType": "container", - "entityUrn": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": { - "platform": "qlik-cloud", - "instance": "qlik_cloud_platform", + "platform": "qlik-sense", + "instance": "qlik_sense_platform", "space": "659d0e41d1b0ecce6eebc9b1", - "type": "shared" + "type": "SpaceType.SHARED" }, "name": "test_space", "description": "", @@ -23,14 +23,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920242978, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322043, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -39,31 +39,31 @@ } }, "systemMetadata": { - "lastObserved": 1705920242980, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322044, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { "json": { - "platform": "urn:li:dataPlatform:qlik-cloud", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" } }, "systemMetadata": { - "lastObserved": 1705920242981, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322045, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -74,14 +74,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920242981, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322045, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -99,64 +99,64 @@ } }, "systemMetadata": { - "lastObserved": 1705920242982, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322046, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" } ] } }, "systemMetadata": { - "lastObserved": 1705920242983, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322047, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": { - "platform": "qlik-cloud", - "instance": "qlik_cloud_platform", + "platform": "qlik-sense", + "instance": "qlik_sense_platform", "space": "personal-space-id", "type": "SpaceType.PERSONAL" }, "name": "personal_space", "description": "", "created": { - "time": 1705920242976 + "time": 1706628321936 }, "lastModified": { - "time": 1705920242976 + "time": 1706628321936 } } }, "systemMetadata": { - "lastObserved": 1705920242995, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322054, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -165,31 +165,31 @@ } }, "systemMetadata": { - "lastObserved": 1705920242997, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322055, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { "json": { - "platform": "urn:li:dataPlatform:qlik-cloud", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" } }, "systemMetadata": { - "lastObserved": 1705920242998, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322056, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -200,42 +200,42 @@ } }, "systemMetadata": { - "lastObserved": 1705920243000, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322057, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" } ] } }, "systemMetadata": { - "lastObserved": 1705920243001, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322057, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": { - "platform": "qlik-cloud", - "instance": "qlik_cloud_platform", + "platform": "qlik-sense", + "instance": "qlik_sense_platform", "app": "b90c4d4e-0d07-4c24-9458-b17d1492660b", "qri": "qri:app:sense://b90c4d4e-0d07-4c24-9458-b17d1492660b", "usage": "ANALYTICS" @@ -251,14 +251,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920243003, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322058, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -267,31 +267,31 @@ } }, "systemMetadata": { - "lastObserved": 1705920243005, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322059, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { "json": { - "platform": "urn:li:dataPlatform:qlik-cloud", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" } }, "systemMetadata": { - "lastObserved": 1705920243007, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322060, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -302,14 +302,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920243008, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322061, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -327,55 +327,236 @@ } }, "systemMetadata": { - "lastObserved": 1705920243008, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322061, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:fdffbfd49b6139680de19df7d14a6737" + "container": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" } }, "systemMetadata": { - "lastObserved": 1705920243010, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322062, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" }, { - "id": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", - "urn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737" + "id": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "urn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" } ] } }, "systemMetadata": { - "lastObserved": 1705920243011, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322063, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1706628322064, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "chartCount": "1" + }, + "title": "New ds sheet", + "description": "", + "charts": [ + "urn:li:chart:(qlik-sense,qlik_sense_platform.QYUUb)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1705296709704, + "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" + }, + "lastModified": { + "time": 1706511226868, + "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1706628322065, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f" + } + }, + "systemMetadata": { + "lastObserved": 1706628322066, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1706628322067, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1706628322067, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "urn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" + }, + { + "id": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", + "urn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1706628322068, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,qlik_sense_platform.QYUUb)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1706628322069, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,qlik_sense_platform.QYUUb)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": {}, + "title": "QYUUb", + "description": "barchart", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1706628322070, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -384,20 +565,20 @@ } }, "systemMetadata": { - "lastObserved": 1705920243015, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322072, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { "schemaName": "659d8aef92df266ef3aa5a7c", - "platform": "urn:li:dataPlatform:qlik-cloud", + "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { "time": 0, @@ -466,14 +647,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920243017, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322073, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -498,47 +679,47 @@ } }, "systemMetadata": { - "lastObserved": 1705920243020, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322075, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:1775d746acf71f0eea10452f850ff472" + "container": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" } }, "systemMetadata": { - "lastObserved": 1705920243021, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322076, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { "json": { - "platform": "urn:li:dataPlatform:qlik-cloud", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" } }, "systemMetadata": { - "lastObserved": 1705920243022, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322077, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -556,14 +737,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920243024, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322077, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -574,46 +755,46 @@ } }, "systemMetadata": { - "lastObserved": 1705920243025, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322078, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" }, { - "id": "urn:li:container:1775d746acf71f0eea10452f850ff472", - "urn": "urn:li:container:1775d746acf71f0eea10452f850ff472" + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" } ] } }, "systemMetadata": { - "lastObserved": 1705920243026, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322079, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": { - "platform": "qlik-cloud", - "instance": "qlik_cloud_platform", + "platform": "qlik-sense", + "instance": "qlik_sense_platform", "app": "e09a68e7-18c9-461d-b957-043e0c045dcd", "qri": "qri:app:sense://e09a68e7-18c9-461d-b957-043e0c045dcd", "usage": "ANALYTICS" @@ -629,14 +810,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920243027, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322080, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -645,31 +826,31 @@ } }, "systemMetadata": { - "lastObserved": 1705920243029, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322081, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { "json": { - "platform": "urn:li:dataPlatform:qlik-cloud", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" } }, "systemMetadata": { - "lastObserved": 1705920243030, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322081, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -680,14 +861,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920243032, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322082, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -705,55 +886,55 @@ } }, "systemMetadata": { - "lastObserved": 1705920243033, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322083, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:1775d746acf71f0eea10452f850ff472" + "container": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" } }, "systemMetadata": { - "lastObserved": 1705920243034, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322084, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" }, { - "id": "urn:li:container:1775d746acf71f0eea10452f850ff472", - "urn": "urn:li:container:1775d746acf71f0eea10452f850ff472" + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" } ] } }, "systemMetadata": { - "lastObserved": 1705920243035, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322084, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -762,20 +943,20 @@ } }, "systemMetadata": { - "lastObserved": 1705920243037, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322086, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { "schemaName": "65a137c849f82a37c625151b", - "platform": "urn:li:dataPlatform:qlik-cloud", + "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { "time": 0, @@ -808,14 +989,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920243038, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322086, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -840,47 +1021,47 @@ } }, "systemMetadata": { - "lastObserved": 1705920243039, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322088, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:fdffbfd49b6139680de19df7d14a6737" + "container": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" } }, "systemMetadata": { - "lastObserved": 1705920243040, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322089, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { "json": { - "platform": "urn:li:dataPlatform:qlik-cloud", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" } }, "systemMetadata": { - "lastObserved": 1705920243041, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322089, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -898,14 +1079,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920243042, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322090, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -916,175 +1097,204 @@ } }, "systemMetadata": { - "lastObserved": 1705920243042, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322091, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" }, { - "id": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", - "urn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737" + "id": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "urn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" } ] } }, "systemMetadata": { - "lastObserved": 1705920243043, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322091, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:1775d746acf71f0eea10452f850ff472", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" } ] } }, "systemMetadata": { - "lastObserved": 1705920243044, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322092, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" } ] } }, "systemMetadata": { - "lastObserved": 1705920243045, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322093, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:f6f5c0da7f27f2b0de030832fa371da2", + "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "urn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1706628322093, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "urn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" }, { - "id": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", - "urn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737" + "id": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", + "urn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f" } ] } }, "systemMetadata": { - "lastObserved": 1705920243046, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322094, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" }, { - "id": "urn:li:container:1775d746acf71f0eea10452f850ff472", - "urn": "urn:li:container:1775d746acf71f0eea10452f850ff472" + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" } ] } }, "systemMetadata": { - "lastObserved": 1705920243046, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322095, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:3317438354ff863161e317b5cb9a2b78", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" }, { - "id": "urn:li:container:1775d746acf71f0eea10452f850ff472", - "urn": "urn:li:container:1775d746acf71f0eea10452f850ff472" + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" } ] } }, "systemMetadata": { - "lastObserved": 1705920243047, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322096, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-cloud,qlik_cloud_platform)" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" }, { - "id": "urn:li:container:fdffbfd49b6139680de19df7d14a6737", - "urn": "urn:li:container:fdffbfd49b6139680de19df7d14a6737" + "id": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "urn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" } ] } }, "systemMetadata": { - "lastObserved": 1705920243048, - "runId": "qlik-cloud-test", + "lastObserved": 1706628322096, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } } diff --git a/metadata-ingestion/tests/integration/qlik_cloud/golden_test_qlik_cloud_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json similarity index 57% rename from metadata-ingestion/tests/integration/qlik_cloud/golden_test_qlik_cloud_ingest.json rename to metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json index dd9edd65061e0..ef7defc2ed589 100644 --- a/metadata-ingestion/tests/integration/qlik_cloud/golden_test_qlik_cloud_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json @@ -1,15 +1,15 @@ [ { "entityType": "container", - "entityUrn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": { - "platform": "qlik-cloud", + "platform": "qlik-sense", "space": "659d0e41d1b0ecce6eebc9b1", - "type": "shared" + "type": "SpaceType.SHARED" }, "name": "test_space", "description": "", @@ -22,14 +22,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920115263, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250099, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -38,30 +38,30 @@ } }, "systemMetadata": { - "lastObserved": 1705920115264, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250100, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { "json": { - "platform": "urn:li:dataPlatform:qlik-cloud" + "platform": "urn:li:dataPlatform:qlik-sense" } }, "systemMetadata": { - "lastObserved": 1705920115265, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250101, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -72,14 +72,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920115265, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250102, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -97,14 +97,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920115266, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250103, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -113,42 +113,42 @@ } }, "systemMetadata": { - "lastObserved": 1705920115267, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250104, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": { - "platform": "qlik-cloud", + "platform": "qlik-sense", "space": "personal-space-id", "type": "SpaceType.PERSONAL" }, "name": "personal_space", "description": "", "created": { - "time": 1705920115262 + "time": 1706628249975 }, "lastModified": { - "time": 1705920115262 + "time": 1706628249975 } } }, "systemMetadata": { - "lastObserved": 1705920115272, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250113, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -157,30 +157,30 @@ } }, "systemMetadata": { - "lastObserved": 1705920115273, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250114, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { "json": { - "platform": "urn:li:dataPlatform:qlik-cloud" + "platform": "urn:li:dataPlatform:qlik-sense" } }, "systemMetadata": { - "lastObserved": 1705920115274, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250115, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -191,14 +191,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920115274, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250116, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -207,20 +207,20 @@ } }, "systemMetadata": { - "lastObserved": 1705920115275, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250117, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": { - "platform": "qlik-cloud", + "platform": "qlik-sense", "app": "b90c4d4e-0d07-4c24-9458-b17d1492660b", "qri": "qri:app:sense://b90c4d4e-0d07-4c24-9458-b17d1492660b", "usage": "ANALYTICS" @@ -236,14 +236,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920115276, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250119, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -252,30 +252,30 @@ } }, "systemMetadata": { - "lastObserved": 1705920115277, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250120, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { "json": { - "platform": "urn:li:dataPlatform:qlik-cloud" + "platform": "urn:li:dataPlatform:qlik-sense" } }, "systemMetadata": { - "lastObserved": 1705920115278, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250121, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -286,14 +286,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920115279, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250122, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -311,51 +311,211 @@ } }, "systemMetadata": { - "lastObserved": 1705920115279, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250122, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:c48b664324e9c8852cc72ac953379725" + "container": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" } }, "systemMetadata": { - "lastObserved": 1705920115280, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250124, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:container:c48b664324e9c8852cc72ac953379725", - "urn": "urn:li:container:c48b664324e9c8852cc72ac953379725" + "id": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "urn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" } ] } }, "systemMetadata": { - "lastObserved": 1705920115281, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250124, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1706628250126, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "chartCount": "1" + }, + "title": "New ds sheet", + "description": "", + "charts": [ + "urn:li:chart:(qlik-sense,QYUUb)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1705296709704, + "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" + }, + "lastModified": { + "time": 1706511226868, + "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1706628250127, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9" + } + }, + "systemMetadata": { + "lastObserved": 1706628250130, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1706628250131, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "urn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" + }, + { + "id": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", + "urn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1706628250133, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,QYUUb)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1706628250136, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,QYUUb)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": {}, + "title": "QYUUb", + "description": "barchart", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1706628250137, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -364,20 +524,20 @@ } }, "systemMetadata": { - "lastObserved": 1705920115282, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250142, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { "schemaName": "659d8aef92df266ef3aa5a7c", - "platform": "urn:li:dataPlatform:qlik-cloud", + "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { "time": 0, @@ -446,14 +606,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920115283, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250144, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -478,30 +638,30 @@ } }, "systemMetadata": { - "lastObserved": 1705920115287, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250149, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4" + "container": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" } }, "systemMetadata": { - "lastObserved": 1705920115288, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250151, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -519,14 +679,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920115288, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250153, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -537,41 +697,41 @@ } }, "systemMetadata": { - "lastObserved": 1705920115289, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250155, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", - "urn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4" + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" } ] } }, "systemMetadata": { - "lastObserved": 1705920115290, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250156, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": { - "platform": "qlik-cloud", + "platform": "qlik-sense", "app": "e09a68e7-18c9-461d-b957-043e0c045dcd", "qri": "qri:app:sense://e09a68e7-18c9-461d-b957-043e0c045dcd", "usage": "ANALYTICS" @@ -587,14 +747,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920115291, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250159, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -603,30 +763,30 @@ } }, "systemMetadata": { - "lastObserved": 1705920115292, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250160, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { "json": { - "platform": "urn:li:dataPlatform:qlik-cloud" + "platform": "urn:li:dataPlatform:qlik-sense" } }, "systemMetadata": { - "lastObserved": 1705920115293, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250161, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -637,14 +797,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920115293, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250162, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -662,51 +822,51 @@ } }, "systemMetadata": { - "lastObserved": 1705920115294, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250163, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4" + "container": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" } }, "systemMetadata": { - "lastObserved": 1705920115295, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250164, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", - "urn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4" + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" } ] } }, "systemMetadata": { - "lastObserved": 1705920115295, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250165, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -715,20 +875,20 @@ } }, "systemMetadata": { - "lastObserved": 1705920115297, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250168, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { "schemaName": "65a137c849f82a37c625151b", - "platform": "urn:li:dataPlatform:qlik-cloud", + "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { "time": 0, @@ -761,14 +921,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920115298, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250170, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -793,30 +953,30 @@ } }, "systemMetadata": { - "lastObserved": 1705920115300, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250173, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:c48b664324e9c8852cc72ac953379725" + "container": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" } }, "systemMetadata": { - "lastObserved": 1705920115301, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250175, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -834,14 +994,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920115302, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250177, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -852,35 +1012,35 @@ } }, "systemMetadata": { - "lastObserved": 1705920115304, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250179, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:container:c48b664324e9c8852cc72ac953379725", - "urn": "urn:li:container:c48b664324e9c8852cc72ac953379725" + "id": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "urn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" } ] } }, "systemMetadata": { - "lastObserved": 1705920115305, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250180, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -889,14 +1049,14 @@ } }, "systemMetadata": { - "lastObserved": 1705920115306, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250182, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:c48b664324e9c8852cc72ac953379725", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -905,92 +1065,117 @@ } }, "systemMetadata": { - "lastObserved": 1705920115307, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250183, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:08c2aa4496d93b7dad8e4f24bc9b1b19", + "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "urn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1706628250185, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:container:c48b664324e9c8852cc72ac953379725", - "urn": "urn:li:container:c48b664324e9c8852cc72ac953379725" + "id": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "urn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" + }, + { + "id": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", + "urn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9" } ] } }, "systemMetadata": { - "lastObserved": 1705920115307, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250188, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", - "urn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4" + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" } ] } }, "systemMetadata": { - "lastObserved": 1705920115309, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250189, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:84c233d296ed77b7769cf6113ec3f0fb", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4", - "urn": "urn:li:container:0e1355c64fe78b2b144dee2dbaaeacc4" + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" } ] } }, "systemMetadata": { - "lastObserved": 1705920115309, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250191, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-cloud,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:container:c48b664324e9c8852cc72ac953379725", - "urn": "urn:li:container:c48b664324e9c8852cc72ac953379725" + "id": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "urn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" } ] } }, "systemMetadata": { - "lastObserved": 1705920115310, - "runId": "qlik-cloud-test", + "lastObserved": 1706628250192, + "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } } diff --git a/metadata-ingestion/tests/integration/qlik_cloud/test_qlik_cloud.py b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py similarity index 82% rename from metadata-ingestion/tests/integration/qlik_cloud/test_qlik_cloud.py rename to metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py index f8b5afa1e56ca..7341c588fc3b3 100644 --- a/metadata-ingestion/tests/integration/qlik_cloud/test_qlik_cloud.py +++ b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py @@ -1,4 +1,5 @@ from typing import Any +from unittest.mock import patch import pytest @@ -587,9 +588,180 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: ) +def mock_websocket_response(*args, **kwargs): + request = kwargs["request"] + if request == { + "jsonrpc": "2.0", + "id": 1, + "handle": -1, + "method": "OpenDoc", + "params": {"qDocName": "b90c4d4e-0d07-4c24-9458-b17d1492660b"}, + }: + return { + "jsonrpc": "2.0", + "id": 1, + "result": { + "qReturn": { + "qType": "Doc", + "qHandle": 1, + "qGenericId": "b90c4d4e-0d07-4c24-9458-b17d1492660b", + } + }, + "change": [1], + } + elif request == { + "jsonrpc": "2.0", + "id": 2, + "handle": 1, + "method": "GetObjects", + "params": {"qOptions": {"qTypes": ["sheet"]}}, + }: + return { + "jsonrpc": "2.0", + "id": 2, + "result": { + "qList": [ + { + "qInfo": { + "qId": "f4f57386-263a-4ec9-b40c-abcd2467f423", + "qType": "sheet", + }, + "qMeta": { + "title": "New ds sheet", + "description": "", + "_resourcetype": "app.object", + "_objecttype": "sheet", + "id": "f4f57386-263a-4ec9-b40c-abcd2467f423", + "approved": False, + "published": False, + "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "ownerId": "657b5abe656297cec3d8b205", + "createdDate": "2024-01-15T11:01:49.704Z", + "modifiedDate": "2024-01-29T12:23:46.868Z", + "publishTime": True, + "privileges": [ + "read", + "update", + "delete", + "publish", + "change_owner", + ], + }, + "qData": {}, + }, + ] + }, + } + elif request == { + "jsonrpc": "2.0", + "id": 3, + "handle": 1, + "method": "GetObject", + "params": {"qId": "f4f57386-263a-4ec9-b40c-abcd2467f423"}, + }: + return { + "jsonrpc": "2.0", + "id": 3, + "result": { + "qReturn": { + "qType": "GenericObject", + "qHandle": 2, + "qGenericType": "sheet", + "qGenericId": "f4f57386-263a-4ec9-b40c-abcd2467f423", + } + }, + } + elif request == { + "jsonrpc": "2.0", + "id": 4, + "handle": 2, + "method": "GetLayout", + "params": {}, + }: + return { + "jsonrpc": "2.0", + "id": 4, + "result": { + "qLayout": { + "qInfo": { + "qId": "f4f57386-263a-4ec9-b40c-abcd2467f423", + "qType": "sheet", + }, + "qMeta": { + "title": "New ds sheet", + "description": "", + "_resourcetype": "app.object", + "_objecttype": "sheet", + "id": "f4f57386-263a-4ec9-b40c-abcd2467f423", + "approved": False, + "published": False, + "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "ownerId": "657b5abe656297cec3d8b205", + "createdDate": "2024-01-15T11:01:49.704Z", + "modifiedDate": "2024-01-29T12:23:46.868Z", + "publishTime": True, + "privileges": [ + "read", + "update", + "delete", + "publish", + "change_owner", + ], + }, + "qSelectionInfo": {}, + "rank": 0, + "thumbnail": {"qStaticContentUrl": {}}, + "columns": 24, + "rows": 12, + "cells": [ + { + "bounds": {"x": 0, "y": 0, "width": 100, "height": 100}, + "col": 0, + "colspan": 24, + "name": "QYUUb", + "row": 0, + "rowspan": 12, + "smartGrid": { + "rowIdx": 0, + "itemIdx": 0, + "item": {"width": 1}, + }, + "type": "barchart", + } + ], + "qChildList": { + "qItems": [ + { + "qInfo": {"qId": "QYUUb", "qType": "barchart"}, + "qMeta": {"privileges": ["read", "update", "delete"]}, + "qData": {"title": ""}, + } + ] + }, + "customRowBase": 12, + "gridResolution": "small", + "layoutOptions": {"mobileLayout": "LIST", "extendable": False}, + "gridMode": "simpleEdit", + } + }, + } + + +@pytest.fixture(scope="module") +def mock_websocket_send_request(): + + with patch( + "datahub.ingestion.source.qlik_sense.qlik_api.QlikAPI._websocket_send_request" + ) as mock_websocket_send_request, patch( + "datahub.ingestion.source.qlik_sense.qlik_api.create_connection" + ): + mock_websocket_send_request.side_effect = mock_websocket_response + yield mock_websocket_send_request + + def default_config(): return { - "tenant_hostname": "https://iq37k6byr9lgam8.us.qlikcloud.com", + "tenant_hostname": "iq37k6byr9lgam8.us.qlikcloud.com", "api_key": "qlik-api-key", "space_pattern": { "allow": [ @@ -601,19 +773,21 @@ def default_config(): @pytest.mark.integration -def test_qlik_cloud_ingest(pytestconfig, tmp_path, requests_mock): +def test_qlik_sense_ingest( + pytestconfig, tmp_path, requests_mock, mock_websocket_send_request +): - test_resources_dir = pytestconfig.rootpath / "tests/integration/qlik_cloud" + test_resources_dir = pytestconfig.rootpath / "tests/integration/qlik_sense" register_mock_api(request_mock=requests_mock) - output_path: str = f"{tmp_path}/qlik_cloud_ingest_mces.json" + output_path: str = f"{tmp_path}/qlik_sense_ingest_mces.json" pipeline = Pipeline.create( { - "run_id": "qlik-cloud-test", + "run_id": "qlik-sense-test", "source": { - "type": "qlik-cloud", + "type": "qlik-sense", "config": { **default_config(), }, @@ -629,7 +803,7 @@ def test_qlik_cloud_ingest(pytestconfig, tmp_path, requests_mock): pipeline.run() pipeline.raise_from_status() - golden_file = "golden_test_qlik_cloud_ingest.json" + golden_file = "golden_test_qlik_sense_ingest.json" mce_helpers.check_golden_file( pytestconfig, @@ -639,9 +813,11 @@ def test_qlik_cloud_ingest(pytestconfig, tmp_path, requests_mock): @pytest.mark.integration -def test_platform_instance_ingest(pytestconfig, tmp_path, requests_mock): +def test_platform_instance_ingest( + pytestconfig, tmp_path, requests_mock, mock_websocket_send_request +): - test_resources_dir = pytestconfig.rootpath / "tests/integration/qlik_cloud" + test_resources_dir = pytestconfig.rootpath / "tests/integration/qlik_sense" register_mock_api(request_mock=requests_mock) @@ -649,12 +825,12 @@ def test_platform_instance_ingest(pytestconfig, tmp_path, requests_mock): pipeline = Pipeline.create( { - "run_id": "qlik-cloud-test", + "run_id": "qlik-sense-test", "source": { - "type": "qlik-cloud", + "type": "qlik-sense", "config": { **default_config(), - "platform_instance": "qlik_cloud_platform", + "platform_instance": "qlik_sense_platform", }, }, "sink": { diff --git a/metadata-service/war/src/main/resources/boot/data_platforms.json b/metadata-service/war/src/main/resources/boot/data_platforms.json index 055771f71a409..6a09670e83e19 100644 --- a/metadata-service/war/src/main/resources/boot/data_platforms.json +++ b/metadata-service/war/src/main/resources/boot/data_platforms.json @@ -586,11 +586,11 @@ } }, { - "urn": "urn:li:dataPlatform:qlik-cloud", + "urn": "urn:li:dataPlatform:qlik-sense", "aspect": { "datasetNameDelimiter": ".", - "name": "qlik-cloud", - "displayName": "Qlik Cloud", + "name": "qlik-sense", + "displayName": "Qlik Sense", "type": "OTHERS", "logoUrl": "/assets/platforms/qliklogo.png" } From a8fcc004c8d011713a6b0c792754b5802d11e1ca Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Fri, 2 Feb 2024 00:51:41 +0530 Subject: [PATCH 06/15] Address review comments and add qlik app used dataset metadata ingestion code --- .../app/ingest/source/builder/sources.json | 2 +- .../sources/qlik-sense/qlik-sense_recipe.yml | 12 +- .../ingestion/source/qlik_sense/config.py | 30 +- .../source/qlik_sense/data_classes.py | 122 ++-- .../ingestion/source/qlik_sense/qlik_api.py | 208 +++---- .../ingestion/source/qlik_sense/qlik_sense.py | 152 +++-- .../source/qlik_sense/websocket_connection.py | 55 ++ ...golden_test_platform_instance_ingest.json} | 452 ++++++++------ .../golden_test_qlik_sense_ingest.json | 431 +++++++------ .../integration/qlik_sense/test_qlik_sense.py | 584 ++++++++---------- 10 files changed, 1111 insertions(+), 937 deletions(-) create mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_sense/websocket_connection.py rename metadata-ingestion/tests/integration/qlik_sense/{golden_test_platform_instace_ingest.json => golden_test_platform_instance_ingest.json} (82%) diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index b18d4e7222741..f4bfd45eb4f83 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -242,6 +242,6 @@ "name": "qlik-sense", "displayName": "Qlik Sense", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/qlik-sense/", - "recipe": "source:\n type: qlik-sense\n config:\n # Coordinates\n tenant_hostname: https://xyz12xz.us.qlikcloud.com\n # Coordinates\n api_key: QLIK_API_KEY\n\n # Optional - filter for certain space names instead of ingesting everything.\n # space_pattern:\n\n # allow:\n # - space_name\n # Whether personal space and apps and datasets should be ingested.\n extract_personal_entity: false\n ingest_owner: true" + "recipe": "source:\n type: qlik-sense\n config:\n # Coordinates\n tenant_hostname: https://xyz12xz.us.qlikcloud.com\n # Coordinates\n api_key: QLIK_API_KEY\n\n # Optional - filter for certain space names instead of ingesting everything.\n # space_pattern:\n\n # allow:\n # - space_name\n ingest_owner: true" } ] diff --git a/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml index 97ab4750b9d09..afcbcc9ed804f 100644 --- a/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml +++ b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml @@ -2,18 +2,24 @@ source: type: qlik-sense config: # Coordinates - tenant_hostname: "https://xyz12xz.us.qlikcloud.com" + tenant_hostname: "xyz12xz.us.qlikcloud.com" # Credentials api_key: "QLIK_API_KEY" # Optional - filter for certain space names instead of ingesting everything. + # Mention 'personal_space' if entities of personal space need to ingest # space_pattern: # allow: # - space_name - # Whether personal space and apps and datasets should be ingested. - extract_personal_entity: false ingest_owner: true + + # Optional -- This mapping is optional and only required to configure platform-instance for Qlik app dataset upstream source table + # A mapping of the Qlik app dataset upstream source to platform instance. Use .. as key. + # app_dataset_source_to_platform_instance: + # ..
: + # platform_instance: cloud_instance + # env: DEV sink: # sink configs diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py index 02ee092f7709e..735d11769963a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py @@ -1,6 +1,6 @@ import logging from dataclasses import dataclass -from typing import Optional +from typing import Dict, Optional import pydantic @@ -42,6 +42,7 @@ class Constant: # Rest API response key constants DATA = "data" + ID = "id" NAME = "name" TYPE = "type" CREATEDAT = "createdAt" @@ -60,7 +61,6 @@ class Constant: SCHEMA = "schema" DATAFIELDS = "dataFields" RESOURCETYPE = "resourceType" - RESOURCEATTRIBUTES = "resourceAttributes" USAGE = "usage" CREATEDDATE = "createdDate" MODIFIEDDATE = "modifiedDate" @@ -77,6 +77,16 @@ class Constant: QITEMS = "qItems" QINFO = "qInfo" QLIST = "qList" + CONNECTORPROPERTIES = "connectorProperties" + TABLEQUALIFIERS = "tableQualifiers" + CONNECTIONINFO = "connectionInfo" + SOURCECONNECTORID = "sourceConnectorID" + DATABASENAME = "databaseName" + SCHEMANAME = "schemaName" + TABLES = "tables" + DATACONNECTORID = "dataconnectorid" + DATACONNECTORNAME = "dataconnectorName" + DATACONNECTORPLATFORM = "dataconnectorPlatform" # Item type APP = "app" DATASET = "dataset" @@ -93,6 +103,10 @@ def report_number_of_spaces(self, number_of_spaces: int) -> None: self.number_of_spaces = number_of_spaces +class PlatformDetail(PlatformInstanceConfigMixin, EnvConfigMixin): + pass + + class QlikSourceConfig( StatefulIngestionConfigBase, PlatformInstanceConfigMixin, EnvConfigMixin ): @@ -101,13 +115,15 @@ class QlikSourceConfig( # Qlik space identifier space_pattern: AllowDenyPattern = pydantic.Field( default=AllowDenyPattern.allow_all(), - description="Regex patterns to filter Qlik spaces in ingestion", - ) - extract_personal_entity: Optional[bool] = pydantic.Field( - default=False, - description="Whether personal space, apps and datasets should be ingested.", + description="Regex patterns to filter Qlik spaces in ingestion." + "Mention 'personal_space' if entities of personal space need to ingest", ) ingest_owner: Optional[bool] = pydantic.Field( default=True, description="Ingest Owner from source. This will override Owner info entered from UI", ) + # Qlik app dataset upstream source to platform instance mapping + app_dataset_source_to_platform_instance: Dict[str, PlatformDetail] = pydantic.Field( + default={}, + description="A mapping of the Qlik app dataset upstream source to platform instance. Use ..
as key.", + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py index 17219636f0173..d3caf80f2bb94 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py @@ -2,7 +2,7 @@ from enum import Enum from typing import Dict, List, Optional, Type, Union -from pydantic import BaseModel, root_validator +from pydantic import BaseModel, Field, root_validator from datahub.emitter.mcp_builder import ContainerKey from datahub.ingestion.source.qlik_sense.config import QLIK_DATETIME_FORMAT, Constant @@ -40,6 +40,11 @@ "BINARY": BytesType, } +KNOWN_DATA_PLATFORM_MAPPING = { + "gbq": "bigquery", + "snowflake": "snowflake", +} + class SpaceKey(ContainerKey): space: str @@ -88,15 +93,53 @@ def update_values(cls, values: Dict) -> Dict: class Item(BaseModel): id: str - name: str - qri: str - description: str + description: str = "" ownerId: str spaceId: str createdAt: datetime updatedAt: datetime +class SchemaField(BaseModel): + name: str + dataType: Optional[str] = None + primaryKey: Optional[bool] = None + nullable: Optional[bool] = None + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.DATATYPE] = values.get(Constant.DATATYPE, {}).get(Constant.TYPE) + return values + + +class QlikDataset(Item): + name: str + secureQri: str + type: str + size: int + rowCount: int + datasetSchema: List[SchemaField] + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + # Update str time to datetime + values[Constant.CREATEDAT] = datetime.strptime( + values[Constant.CREATEDTIME], QLIK_DATETIME_FORMAT + ) + values[Constant.UPDATEDAT] = datetime.strptime( + values[Constant.LASTMODIFIEDTIME], QLIK_DATETIME_FORMAT + ) + if not values.get(Constant.SPACEID): + # spaceId none indicates dataset present in personal space + values[Constant.SPACEID] = Constant.PERSONAL_SPACE_ID + values[Constant.QRI] = values[Constant.SECUREQRI] + values[Constant.SIZE] = values[Constant.OPERATIONAL].get(Constant.SIZE, 0) + values[Constant.ROWCOUNT] = values[Constant.OPERATIONAL][Constant.ROWCOUNT] + + values[Constant.DATASETSCHEMA] = values[Constant.SCHEMA][Constant.DATAFIELDS] + return values + + class Chart(BaseModel): qId: str qType: str @@ -122,61 +165,48 @@ def update_values(cls, values: Dict) -> Dict: return values -class App(Item): - usage: str - sheets: List[Sheet] = [] +class QlikAppDataset(BaseModel): + tableName: str + schemaName: str + databaseName: str + tableAlias: str + dataconnectorid: str + dataconnectorName: str + dataconnectorPlatform: str + datasetSchema: List[SchemaField] = Field(alias="fields") @root_validator(pre=True) def update_values(cls, values: Dict) -> Dict: - values[Constant.CREATEDAT] = datetime.strptime( - values[Constant.CREATEDDATE], QLIK_DATETIME_FORMAT - ) - values[Constant.UPDATEDAT] = datetime.strptime( - values[Constant.MODIFIEDDATE], QLIK_DATETIME_FORMAT - ) - if not values.get(Constant.SPACEID): - # spaceId none indicates app present in personal space - values[Constant.SPACEID] = Constant.PERSONAL_SPACE_ID - values[Constant.QRI] = f"qri:app:sense://{values['id']}" + values[Constant.DATABASENAME] = values[Constant.CONNECTORPROPERTIES][ + Constant.TABLEQUALIFIERS + ][0] + values[Constant.SCHEMANAME] = values[Constant.CONNECTORPROPERTIES][ + Constant.TABLEQUALIFIERS + ][1] + values[Constant.DATACONNECTORID] = values[Constant.CONNECTIONINFO][Constant.ID] + values[Constant.DATACONNECTORPLATFORM] = values[Constant.CONNECTIONINFO][ + Constant.SOURCECONNECTORID + ] return values -class SchemaField(BaseModel): - name: str - dataType: str - primaryKey: bool - nullable: bool - - -class QlikDataset(Item): - type: str - size: int - rowCount: int - datasetSchema: List[SchemaField] +class App(Item): + qTitle: str + qri: str + qUsage: str + sheets: List[Sheet] = [] + datasets: List[QlikAppDataset] = [] @root_validator(pre=True) def update_values(cls, values: Dict) -> Dict: - # Update str time to datetime values[Constant.CREATEDAT] = datetime.strptime( - values[Constant.CREATEDTIME], QLIK_DATETIME_FORMAT + values[Constant.CREATEDDATE], QLIK_DATETIME_FORMAT ) values[Constant.UPDATEDAT] = datetime.strptime( - values[Constant.LASTMODIFIEDTIME], QLIK_DATETIME_FORMAT + values[Constant.MODIFIEDDATE], QLIK_DATETIME_FORMAT ) if not values.get(Constant.SPACEID): - # spaceId none indicates dataset present in personal space + # spaceId none indicates app present in personal space values[Constant.SPACEID] = Constant.PERSONAL_SPACE_ID - values[Constant.QRI] = values[Constant.SECUREQRI] - values[Constant.SIZE] = values[Constant.OPERATIONAL].get(Constant.SIZE, 0) - values[Constant.ROWCOUNT] = values[Constant.OPERATIONAL][Constant.ROWCOUNT] - - values[Constant.DATASETSCHEMA] = [ - { - Constant.NAME: field[Constant.NAME], - Constant.DATATYPE: field[Constant.DATATYPE][Constant.TYPE], - Constant.PRIMARYKEY: field[Constant.PRIMARYKEY], - Constant.NULLABLE: field[Constant.NULLABLE], - } - for field in values[Constant.SCHEMA][Constant.DATAFIELDS] - ] + values[Constant.QRI] = f"qri:app:sense://{values['id']}" return values diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py index 6abc4c931cf9c..267d2d424fa69 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py @@ -1,20 +1,21 @@ -import json import logging import sys from typing import Any, Dict, List, Optional import requests -from websocket import WebSocket, create_connection from datahub.ingestion.source.qlik_sense.config import Constant, QlikSourceConfig from datahub.ingestion.source.qlik_sense.data_classes import ( + PERSONAL_SPACE_DICT, App, Chart, Item, + QlikAppDataset, QlikDataset, Sheet, Space, ) +from datahub.ingestion.source.qlik_sense.websocket_connection import WebsocketConnection # Logger instance logger = logging.getLogger(__name__) @@ -22,6 +23,7 @@ class QlikAPI: def __init__(self, config: QlikSourceConfig) -> None: + self.spaces: Dict = {} self.config = config self.session = requests.Session() self.session.headers.update( @@ -31,7 +33,6 @@ def __init__(self, config: QlikSourceConfig) -> None: } ) self.rest_api_url = f"https://{self.config.tenant_hostname}/api/v1" - self.websocket_url = f"wss://{self.config.tenant_hostname}/app" # Test connection by fetching list of api keys logger.info("Trying to connect to {}".format(self.rest_api_url)) self.session.get(f"{self.rest_api_url}/api-keys").raise_for_status() @@ -44,36 +45,20 @@ def _log_http_error(self, message: str) -> Any: logger.debug(msg=message, exc_info=e) return e - def _websocket_send_request( - self, socket_connection: WebSocket, request: dict - ) -> Dict: - """ - Method to send request to websocket - """ - socket_connection.send(json.dumps(request)) - resp = socket_connection.recv() - if request["handle"] == -1: - resp = socket_connection.recv() - return json.loads(resp) - - def _get_websocket_request_dict( - self, id: int, handle: int, method: str, params: Dict = {} - ) -> Dict: - return { - "jsonrpc": "2.0", - "id": id, - "handle": handle, - "method": method, - "params": params, - } - def get_spaces(self) -> List[Space]: spaces: List[Space] = [] try: response = self.session.get(f"{self.rest_api_url}/spaces") response.raise_for_status() for space_dict in response.json()[Constant.DATA]: - spaces.append(Space.parse_obj(space_dict)) + space = Space.parse_obj(space_dict) + spaces.append(space) + self.spaces[space.id] = space.name + # Add personal space entity + spaces.append(Space.parse_obj(PERSONAL_SPACE_DICT)) + self.spaces[PERSONAL_SPACE_DICT[Constant.ID]] = PERSONAL_SPACE_DICT[ + Constant.NAME + ] except Exception: self._log_http_error(message="Unable to fetch spaces") return spaces @@ -89,57 +74,17 @@ def _get_dataset(self, dataset_id: str) -> Optional[QlikDataset]: ) return None - def get_items(self) -> List[Item]: - items: List[Item] = [] - try: - response = self.session.get(f"{self.rest_api_url}/items") - response.raise_for_status() - data = response.json()[Constant.DATA] - for item in data: - resource_type = item[Constant.RESOURCETYPE] - resource_attributes = item[Constant.RESOURCEATTRIBUTES] - if resource_type == Constant.APP: - app = App.parse_obj(resource_attributes) - app.sheets = self._get_app_sheets(app_id=app.id) - items.append(app) - elif resource_type == Constant.DATASET: - dataset = self._get_dataset(dataset_id=item[Constant.RESOURCEID]) - if dataset: - items.append(dataset) - - except Exception: - self._log_http_error(message="Unable to fetch items") - return items - def _get_sheet( self, - socket_connection: WebSocket, - request_id: int, - current_handle: int, + websocket_connection: WebsocketConnection, sheet_id: str, ) -> Optional[Sheet]: try: - response = self._websocket_send_request( - socket_connection=socket_connection, - request=self._get_websocket_request_dict( - id=request_id, - handle=current_handle, - method="GetObject", - params={"qId": sheet_id}, - ), - ) - request_id += 1 - current_handle = response[Constant.RESULT][Constant.QRETURN][ - Constant.QHANDLE - ] - response = self._websocket_send_request( - socket_connection=socket_connection, - request=self._get_websocket_request_dict( - id=request_id, handle=current_handle, method="GetLayout" - ), + websocket_connection.websocket_send_request( + method="GetObject", params={"qId": sheet_id} ) - request_id += 1 - sheet_dict = response[Constant.RESULT][Constant.QLAYOUT] + response = websocket_connection.websocket_send_request(method="GetLayout") + sheet_dict = response[Constant.QLAYOUT] sheet = Sheet.parse_obj(sheet_dict[Constant.QMETA]) for chart_dict in sheet_dict[Constant.QCHILDLIST][Constant.QITEMS]: sheet.charts.append(Chart.parse_obj(chart_dict[Constant.QINFO])) @@ -148,49 +93,98 @@ def _get_sheet( self._log_http_error(message=f"Unable to fetch sheet with id {sheet_id}") return None - def _get_app_sheets(self, app_id: str) -> List[Sheet]: - sheets: List[Sheet] = [] - request_id = 1 - current_handle = -1 + def _get_app_used_datasets( + self, websocket_connection: WebsocketConnection, app_id: str + ) -> List[QlikAppDataset]: + datasets: List[QlikAppDataset] = [] try: - socket_connection = create_connection( - f"{self.websocket_url}/{app_id}", - header={"Authorization": f"Bearer {self.config.api_key}"}, + websocket_connection.websocket_send_request( + method="GetObject", + params=["LoadModel"], ) - response = self._websocket_send_request( - socket_connection=socket_connection, - request=self._get_websocket_request_dict( - id=request_id, - handle=current_handle, - method="OpenDoc", - params={"qDocName": app_id}, - ), + response = websocket_connection.websocket_send_request(method="GetLayout") + for table_dict in response[Constant.QLAYOUT][Constant.TABLES]: + # Condition to Add connection based table only + if table_dict["boxType"] == "blackbox": + datasets.append(QlikAppDataset.parse_obj(table_dict)) + websocket_connection.handle.pop() + except Exception: + self._log_http_error( + message=f"Unable to fetch app used datasets for app {app_id}" ) - request_id += 1 - current_handle = response["result"]["qReturn"]["qHandle"] - response = self._websocket_send_request( - socket_connection=socket_connection, - request=self._get_websocket_request_dict( - id=request_id, - handle=current_handle, - method="GetObjects", - params={ - "qOptions": { - "qTypes": ["sheet"], - } - }, - ), + return datasets + + def _get_app_sheets( + self, websocket_connection: WebsocketConnection, app_id: str + ) -> List[Sheet]: + sheets: List[Sheet] = [] + try: + response = websocket_connection.websocket_send_request( + method="GetObjects", + params={ + "qOptions": { + "qTypes": ["sheet"], + } + }, ) - request_id += 1 - for sheet_dict in response[Constant.RESULT][Constant.QLIST]: + for sheet_dict in response[Constant.QLIST]: sheet = self._get_sheet( - socket_connection=socket_connection, - request_id=request_id, - current_handle=current_handle, + websocket_connection=websocket_connection, sheet_id=sheet_dict[Constant.QINFO][Constant.QID], ) if sheet: sheets.append(sheet) + websocket_connection.handle.pop() except Exception: - self._log_http_error(message="Unable to fetch sheets") + self._log_http_error(message=f"Unable to fetch sheets for app {app_id}") return sheets + + def _get_app(self, app_id: str) -> Optional[App]: + try: + websocket_connection = WebsocketConnection( + self.config.tenant_hostname, self.config.api_key, app_id + ) + websocket_connection.websocket_send_request( + method="OpenDoc", + params={"qDocName": app_id}, + ) + response = websocket_connection.websocket_send_request( + method="GetAppLayout" + ) + app = App.parse_obj(response[Constant.QLAYOUT]) + app.sheets = self._get_app_sheets(websocket_connection, app_id) + app.datasets = self._get_app_used_datasets(websocket_connection, app_id) + websocket_connection.close_websocket() + return app + except Exception: + self._log_http_error(message=f"Unable to fetch app with id {app_id}") + return None + + def get_items(self) -> List[Item]: + items: List[Item] = [] + try: + response = self.session.get(f"{self.rest_api_url}/items") + response.raise_for_status() + data = response.json()[Constant.DATA] + for item in data: + # spaceId none indicates item present in personal space + if not item.get(Constant.SPACEID): + item[Constant.SPACEID] = Constant.PERSONAL_SPACE_ID + if self.config.space_pattern.allowed( + self.spaces[item[Constant.SPACEID]] + ): + resource_type = item[Constant.RESOURCETYPE] + if resource_type == Constant.APP: + app = self._get_app(app_id=item[Constant.RESOURCEID]) + if app: + items.append(app) + elif resource_type == Constant.DATASET: + dataset = self._get_dataset( + dataset_id=item[Constant.RESOURCEID] + ) + if dataset: + items.append(dataset) + + except Exception: + self._log_http_error(message="Unable to fetch items") + return items diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py index 535b4f5b7a565..d1fc2b88a7477 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py @@ -28,15 +28,17 @@ ) from datahub.ingestion.source.qlik_sense.config import ( Constant, + PlatformDetail, QlikSourceConfig, QlikSourceReport, ) from datahub.ingestion.source.qlik_sense.data_classes import ( FIELD_TYPE_MAPPING, - PERSONAL_SPACE_DICT, + KNOWN_DATA_PLATFORM_MAPPING, App, AppKey, Chart, + QlikAppDataset, QlikDataset, SchemaField as QlikDatasetSchemaField, Sheet, @@ -55,7 +57,12 @@ SubTypes, TimeStamp, ) -from datahub.metadata.com.linkedin.pegasus2avro.dataset import DatasetProperties +from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( + DatasetLineageType, + DatasetProperties, + Upstream, + UpstreamLineage, +) from datahub.metadata.com.linkedin.pegasus2avro.schema import ( MySqlDDL, NullType, @@ -142,9 +149,6 @@ def _gen_app_key(self, app_id: str) -> AppKey: instance=self.config.platform_instance, ) - def _gen_personal_space(self) -> Space: - return Space.parse_obj(PERSONAL_SPACE_DICT) - def _get_audit_stamp(self, date: datetime, username: str) -> AuditStampClass: return AuditStampClass( time=int(date.timestamp() * 1000), @@ -158,10 +162,7 @@ def _get_allowed_spaces(self) -> List[Space]: for space in all_spaces if self.config.space_pattern.allowed(space.name) ] - # Add personal space entity if flag is enable - if self.config.extract_personal_entity: - allowed_spaces.append(self._gen_personal_space()) - logger.info(f"Number of spaces = {len(allowed_spaces)}") + logger.info(f"Number of spaces = {len(all_spaces)}") self.reporter.report_number_of_spaces(len(all_spaces)) logger.info(f"Number of allowed spaces = {len(allowed_spaces)}") return allowed_spaces @@ -235,9 +236,7 @@ def _gen_dashboard_info_workunit(self, sheet: Sheet) -> MetadataWorkUnit: entityUrn=dashboard_urn, aspect=dashboard_info_cls ).as_workunit() - def _gen_charts_workunit( - self, charts: List[Chart], space_id: str, app_id: str, sheet_id: str - ) -> Iterable[MetadataWorkUnit]: + def _gen_charts_workunit(self, charts: List[Chart]) -> Iterable[MetadataWorkUnit]: """ Map Qlik Chart to Datahub Chart """ @@ -259,13 +258,6 @@ def _gen_charts_workunit( ), ).as_workunit() - # yield MetadataChangeProposalWrapper( - # entityUrn=chart_urn, - # aspect=BrowsePathsClass( - # paths=[f"/{self.platform}/{space_id}/{app_id}/{sheet_id}"] - # ), - # ).as_workunit() - def _gen_sheets_workunit( self, sheets: List[Sheet], space_id: str, app_id: str ) -> Iterable[MetadataWorkUnit]: @@ -292,21 +284,104 @@ def _gen_sheets_workunit( if self.config.ingest_owner: yield self._gen_entity_owner_aspect(dashboard_urn, sheet.ownerId) - yield from self._gen_charts_workunit( - sheet.charts, space_id, app_id, sheet.id + yield from self._gen_charts_workunit(sheet.charts) + + def _gen_app_dataset_upstream_lineage( + self, dataset: QlikAppDataset + ) -> MetadataWorkUnit: + dataset_identifier = self._get_app_dataset_identifier(dataset) + dataset_urn = self._gen_qlik_dataset_urn(dataset_identifier) + upstream_dataset_platform_detail = ( + self.config.app_dataset_source_to_platform_instance.get( + dataset_identifier, + PlatformDetail( + env=self.config.env, platform_instance=self.config.platform_instance + ), + ) + ) + + upstream_dataset_urn = builder.make_dataset_urn_with_platform_instance( + name=dataset_identifier, + platform=KNOWN_DATA_PLATFORM_MAPPING.get( + dataset.dataconnectorPlatform, dataset.dataconnectorPlatform + ), + env=upstream_dataset_platform_detail.env, + platform_instance=upstream_dataset_platform_detail.platform_instance, + ) + + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=UpstreamLineage( + upstreams=[ + Upstream(dataset=upstream_dataset_urn, type=DatasetLineageType.COPY) + ] + ), + ).as_workunit() + + def _gen_app_dataset_properties(self, dataset: QlikAppDataset) -> MetadataWorkUnit: + dataset_urn = self._gen_qlik_dataset_urn( + self._get_app_dataset_identifier(dataset) + ) + + dataset_properties = DatasetProperties( + name=dataset.tableName, + qualifiedName=dataset.tableAlias, + ) + dataset_properties.customProperties.update( + { + Constant.DATACONNECTORID: dataset.dataconnectorid, + Constant.DATACONNECTORNAME: dataset.dataconnectorName, + } + ) + + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=dataset_properties + ).as_workunit() + + def _get_app_dataset_identifier(self, dataset: QlikAppDataset) -> str: + return f"{dataset.databaseName}.{dataset.schemaName}.{dataset.tableName}" + + def _gen_app_dataset_workunit( + self, datasets: List[QlikAppDataset], app_id: str + ) -> Iterable[MetadataWorkUnit]: + for dataset in datasets: + dataset_identifier = self._get_app_dataset_identifier(dataset) + dataset_urn = self._gen_qlik_dataset_urn(dataset_identifier) + + yield self._gen_entity_status_aspect(dataset_urn) + + yield self._gen_schema_metadata(dataset_identifier, dataset.datasetSchema) + + yield self._gen_app_dataset_properties(dataset) + + yield from add_entity_to_container( + container_key=self._gen_app_key(app_id), + entity_type="dataset", + entity_urn=dataset_urn, ) + dpi_aspect = self._gen_dataplatform_instance_aspect(dataset_urn) + if dpi_aspect: + yield dpi_aspect + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=SubTypes(typeNames=[DatasetSubTypes.QLIK_DATASET]), + ).as_workunit() + + yield self._gen_app_dataset_upstream_lineage(dataset) + def _gen_app_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: """ Map Qlik App to Datahub container """ yield from gen_containers( container_key=self._gen_app_key(app.id), - name=app.name, + name=app.qTitle, description=app.description, sub_types=[BIContainerSubTypes.QLIK_APP], parent_container_key=self._gen_space_key(app.spaceId), - extra_properties={Constant.QRI: app.qri, Constant.USAGE: app.usage}, + extra_properties={Constant.QRI: app.qri, Constant.USAGE: app.qUsage}, owner_urn=builder.make_user_urn(app.ownerId) if self.config.ingest_owner else None, @@ -314,8 +389,9 @@ def _gen_app_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: last_modified=int(app.updatedAt.timestamp() * 1000), ) yield from self._gen_sheets_workunit(app.sheets, app.spaceId, app.id) + yield from self._gen_app_dataset_workunit(app.datasets, app.id) - def _gen_dataset_urn(self, dataset_identifier: str) -> str: + def _gen_qlik_dataset_urn(self, dataset_identifier: str) -> str: return builder.make_dataset_urn_with_platform_instance( name=dataset_identifier, env=self.config.env, @@ -348,25 +424,29 @@ def _gen_schema_fields( fieldPath=field.name, type=SchemaFieldDataTypeClass( type=FIELD_TYPE_MAPPING.get(field.dataType, NullType)() + if field.dataType + else NullType() ), # NOTE: nativeDataType will not be in sync with older connector - nativeDataType=field.dataType, + nativeDataType=field.dataType if field.dataType else "", nullable=field.nullable, isPartOfKey=field.primaryKey, ) schema_fields.append(schema_field) return schema_fields - def _gen_schema_metadata(self, dataset: QlikDataset) -> MetadataWorkUnit: - dataset_urn = self._gen_dataset_urn(dataset.id) + def _gen_schema_metadata( + self, dataset_identifier: str, dataset_schema: List[QlikDatasetSchemaField] + ) -> MetadataWorkUnit: + dataset_urn = self._gen_qlik_dataset_urn(dataset_identifier) schema_metadata = SchemaMetadata( - schemaName=dataset.id, + schemaName=dataset_identifier, platform=builder.make_data_platform_urn(self.platform), version=0, hash="", platformSchema=MySqlDDL(tableSchema=""), - fields=self._gen_schema_fields(dataset.datasetSchema), + fields=self._gen_schema_fields(dataset_schema), ) return MetadataChangeProposalWrapper( @@ -374,7 +454,7 @@ def _gen_schema_metadata(self, dataset: QlikDataset) -> MetadataWorkUnit: ).as_workunit() def _gen_dataset_properties(self, dataset: QlikDataset) -> MetadataWorkUnit: - dataset_urn = self._gen_dataset_urn(dataset.id) + dataset_urn = self._gen_qlik_dataset_urn(dataset.id) dataset_properties = DatasetProperties( name=dataset.name, @@ -385,7 +465,7 @@ def _gen_dataset_properties(self, dataset: QlikDataset) -> MetadataWorkUnit: ) dataset_properties.customProperties.update( { - Constant.QRI: dataset.qri, + Constant.QRI: dataset.secureQri, Constant.SPACEID: dataset.spaceId, Constant.TYPE: dataset.type, Constant.SIZE: str(dataset.size), @@ -398,11 +478,11 @@ def _gen_dataset_properties(self, dataset: QlikDataset) -> MetadataWorkUnit: ).as_workunit() def _gen_dataset_workunit(self, dataset: QlikDataset) -> Iterable[MetadataWorkUnit]: - dataset_urn = self._gen_dataset_urn(dataset.id) + dataset_urn = self._gen_qlik_dataset_urn(dataset.id) yield self._gen_entity_status_aspect(dataset_urn) - yield self._gen_schema_metadata(dataset) + yield self._gen_schema_metadata(dataset.id, dataset.datasetSchema) yield self._gen_dataset_properties(dataset) @@ -440,12 +520,6 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: for space in self._get_allowed_spaces(): yield from self._gen_space_workunit(space) for item in self.qlik_api.get_items(): - # If item is personal item and flag is Disable, skip ingesting personal item - if ( - item.spaceId == Constant.PERSONAL_SPACE_ID - and not self.config.extract_personal_entity - ): - continue if isinstance(item, App): yield from self._gen_app_workunit(item) elif isinstance(item, QlikDataset): diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/websocket_connection.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/websocket_connection.py new file mode 100644 index 0000000000000..01ca9415f886a --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/websocket_connection.py @@ -0,0 +1,55 @@ +import json +from typing import Dict, List, Optional, Union + +from websocket import WebSocket, create_connection + +from datahub.ingestion.source.qlik_sense.config import Constant + + +class WebsocketConnection: + def __init__(self, tenant_hostname: str, api_key: str, app_id: str) -> None: + self.websocket_url = f"wss://{tenant_hostname}/app" + self.socket_connection: Optional[WebSocket] = create_connection( + f"{self.websocket_url}/{app_id}", + header={"Authorization": f"Bearer {api_key}"}, + ) + self.request_id = 0 + self.handle = [-1] + + def _build_websocket_request_dict( + self, method: str, params: Union[Dict, List] = {} + ) -> Dict: + return { + "jsonrpc": "2.0", + "id": self.request_id, + "handle": self.handle[-1], + "method": method, + "params": params, + } + + def _send_request(self, request: Dict) -> Dict: + if self.socket_connection: + self.socket_connection.send(json.dumps(request)) + resp = self.socket_connection.recv() + if request["handle"] == -1: + resp = self.socket_connection.recv() + return json.loads(resp)[Constant.RESULT] + return {} + + def websocket_send_request( + self, method: str, params: Union[Dict, List] = {} + ) -> Dict: + """ + Method to send request to websocket + """ + self.request_id += 1 + request = self._build_websocket_request_dict(method, params) + response = self._send_request(request=request) + handle = response.get(Constant.QRETURN, {}).get(Constant.QHANDLE) + if handle: + self.handle.append(handle) + return response + + def close_websocket(self) -> None: + if self.socket_connection: + self.socket_connection.close() diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instace_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json similarity index 82% rename from metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instace_ingest.json rename to metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json index f991105f0a5a1..d3864749f0179 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instace_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json @@ -23,7 +23,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322043, + "lastObserved": 1706814884085, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -39,7 +39,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322044, + "lastObserved": 1706814884086, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -56,7 +56,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322045, + "lastObserved": 1706814884087, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -74,7 +74,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322045, + "lastObserved": 1706814884087, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -99,7 +99,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322046, + "lastObserved": 1706814884088, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -120,7 +120,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322047, + "lastObserved": 1706814884089, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -141,15 +141,15 @@ "name": "personal_space", "description": "", "created": { - "time": 1706628321936 + "time": 1706814883987 }, "lastModified": { - "time": 1706628321936 + "time": 1706814883987 } } }, "systemMetadata": { - "lastObserved": 1706628322054, + "lastObserved": 1706814884096, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -165,7 +165,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322055, + "lastObserved": 1706814884097, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -182,7 +182,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322056, + "lastObserved": 1706814884098, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -200,7 +200,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322057, + "lastObserved": 1706814884098, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -221,14 +221,14 @@ } }, "systemMetadata": { - "lastObserved": 1706628322057, + "lastObserved": 1706814884099, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { @@ -236,29 +236,29 @@ "customProperties": { "platform": "qlik-sense", "instance": "qlik_sense_platform", - "app": "b90c4d4e-0d07-4c24-9458-b17d1492660b", - "qri": "qri:app:sense://b90c4d4e-0d07-4c24-9458-b17d1492660b", + "app": "e09a68e7-18c9-461d-b957-043e0c045dcd", + "qri": "qri:app:sense://e09a68e7-18c9-461d-b957-043e0c045dcd", "usage": "ANALYTICS" }, - "name": "test_app_1", + "name": "IPL_Matches_2022", "description": "", "created": { - "time": 1704769768928 + "time": 1704803736545 }, "lastModified": { - "time": 1704771240100 + "time": 1705297020333 } } }, "systemMetadata": { - "lastObserved": 1706628322058, + "lastObserved": 1706814884100, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -267,14 +267,14 @@ } }, "systemMetadata": { - "lastObserved": 1706628322059, + "lastObserved": 1706814884101, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -284,14 +284,14 @@ } }, "systemMetadata": { - "lastObserved": 1706628322060, + "lastObserved": 1706814884102, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -302,14 +302,14 @@ } }, "systemMetadata": { - "lastObserved": 1706628322061, + "lastObserved": 1706814884102, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -327,30 +327,30 @@ } }, "systemMetadata": { - "lastObserved": 1706628322061, + "lastObserved": 1706814884103, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" + "container": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" } }, "systemMetadata": { - "lastObserved": 1706628322062, + "lastObserved": 1706814884104, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -361,14 +361,14 @@ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" }, { - "id": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", - "urn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" } ] } }, "systemMetadata": { - "lastObserved": 1706628322063, + "lastObserved": 1706814884104, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -384,7 +384,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322064, + "lastObserved": 1706814884106, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -418,7 +418,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322065, + "lastObserved": 1706814884106, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -430,11 +430,11 @@ "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f" + "container": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" } }, "systemMetadata": { - "lastObserved": 1706628322066, + "lastObserved": 1706814884107, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -451,7 +451,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322067, + "lastObserved": 1706814884108, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -476,7 +476,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322067, + "lastObserved": 1706814884109, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -494,18 +494,18 @@ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" }, { - "id": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", - "urn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" }, { - "id": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", - "urn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f" + "id": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "urn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" } ] } }, "systemMetadata": { - "lastObserved": 1706628322068, + "lastObserved": 1706814884109, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -521,7 +521,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322069, + "lastObserved": 1706814884110, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -549,14 +549,14 @@ } }, "systemMetadata": { - "lastObserved": 1706628322070, + "lastObserved": 1706814884111, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -565,19 +565,19 @@ } }, "systemMetadata": { - "lastObserved": 1706628322072, + "lastObserved": 1706814884113, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "659d8aef92df266ef3aa5a7c", + "schemaName": "harshal-playground-306419.test_dataset.test_table", "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { @@ -596,50 +596,14 @@ }, "fields": [ { - "fieldPath": "ID", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "City", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Date", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Season", + "fieldPath": "name", "nullable": false, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.NullType": {} } }, - "nativeDataType": "INTEGER", + "nativeDataType": "", "recursive": false, "isPartOfKey": false } @@ -647,62 +611,52 @@ } }, "systemMetadata": { - "lastObserved": 1706628322073, + "lastObserved": 1706814884114, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { "json": { "customProperties": { - "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", - "spaceId": "659d0e41d1b0ecce6eebc9b1", - "type": "DELIMETED", - "size": "38168", - "rowCount": "74" - }, - "name": "IPL_Matches_2022.csv", - "qualifiedName": "IPL_Matches_2022.csv", - "description": "", - "created": { - "time": 1704803735527 - }, - "lastModified": { - "time": 1704804512453 + "dataconnectorid": "bb5be407-d3d3-4f19-858c-e71d593f09ae", + "dataconnectorName": "Google_BigQuery_harshal-playground-306419" }, + "name": "test_table", + "qualifiedName": "test_table", "tags": [] } }, "systemMetadata": { - "lastObserved": 1706628322075, + "lastObserved": 1706814884115, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + "container": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" } }, "systemMetadata": { - "lastObserved": 1706628322076, + "lastObserved": 1706814884116, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -712,57 +666,57 @@ } }, "systemMetadata": { - "lastObserved": 1706628322077, + "lastObserved": 1706814884117, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "subTypes", "aspect": { "json": { - "owners": [ - { - "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "typeNames": [ + "Qlik Dataset" + ] } }, "systemMetadata": { - "lastObserved": 1706628322077, + "lastObserved": 1706814884117, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "upstreamLineage", "aspect": { "json": { - "typeNames": [ - "Qlik Dataset" + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,google-cloud.harshal-playground-306419.test_dataset.test_table,DEV)", + "type": "COPY" + } ] } }, "systemMetadata": { - "lastObserved": 1706628322078, + "lastObserved": 1706814884118, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -775,100 +729,186 @@ { "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "urn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" } ] } }, "systemMetadata": { - "lastObserved": 1706628322079, + "lastObserved": 1706814884119, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "platform": "qlik-sense", - "instance": "qlik_sense_platform", - "app": "e09a68e7-18c9-461d-b957-043e0c045dcd", - "qri": "qri:app:sense://e09a68e7-18c9-461d-b957-043e0c045dcd", - "usage": "ANALYTICS" - }, - "name": "IPL_Matches_2022", - "description": "", + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1706814884120, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "659d8aef92df266ef3aa5a7c", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, "created": { - "time": 1704803736545 + "time": 0, + "actor": "urn:li:corpuser:unknown" }, "lastModified": { - "time": 1705297014684 - } + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Season", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] } }, "systemMetadata": { - "lastObserved": 1706628322080, + "lastObserved": 1706814884121, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "datasetProperties", "aspect": { "json": { - "removed": false + "customProperties": { + "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "type": "DELIMETED", + "size": "38168", + "rowCount": "74" + }, + "name": "IPL_Matches_2022.csv", + "qualifiedName": "IPL_Matches_2022.csv", + "description": "", + "created": { + "time": 1704803735527 + }, + "lastModified": { + "time": 1704804512453 + }, + "tags": [] } }, "systemMetadata": { - "lastObserved": 1706628322081, + "lastObserved": 1706814884123, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "container", "aspect": { "json": { - "platform": "urn:li:dataPlatform:qlik-sense", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + "container": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" } }, "systemMetadata": { - "lastObserved": 1706628322081, + "lastObserved": 1706814884124, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "typeNames": [ - "Qlik App" - ] + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" } }, "systemMetadata": { - "lastObserved": 1706628322082, + "lastObserved": 1706814884125, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -886,30 +926,32 @@ } }, "systemMetadata": { - "lastObserved": 1706628322083, + "lastObserved": 1706814884126, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + "typeNames": [ + "Qlik Dataset" + ] } }, "systemMetadata": { - "lastObserved": 1706628322084, + "lastObserved": 1706814884126, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -927,7 +969,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322084, + "lastObserved": 1706814884127, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -943,7 +985,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322086, + "lastObserved": 1706814884128, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -989,7 +1031,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322086, + "lastObserved": 1706814884129, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1021,7 +1063,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322088, + "lastObserved": 1706814884130, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1037,7 +1079,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322089, + "lastObserved": 1706814884131, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1054,7 +1096,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322089, + "lastObserved": 1706814884132, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1079,7 +1121,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322090, + "lastObserved": 1706814884133, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1097,7 +1139,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322091, + "lastObserved": 1706814884134, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1122,7 +1164,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322091, + "lastObserved": 1706814884134, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1143,7 +1185,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322092, + "lastObserved": 1706814884135, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1164,14 +1206,14 @@ } }, "systemMetadata": { - "lastObserved": 1706628322093, + "lastObserved": 1706814884136, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", + "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1182,14 +1224,14 @@ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" }, { - "id": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", - "urn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" } ] } }, "systemMetadata": { - "lastObserved": 1706628322093, + "lastObserved": 1706814884136, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1207,25 +1249,25 @@ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" }, { - "id": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", - "urn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" }, { - "id": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f", - "urn": "urn:li:container:94950c4125e79a1d8a00ecf75fef861f" + "id": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "urn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" } ] } }, "systemMetadata": { - "lastObserved": 1706628322094, + "lastObserved": 1706814884137, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1238,19 +1280,23 @@ { "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "urn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" } ] } }, "systemMetadata": { - "lastObserved": 1706628322095, + "lastObserved": 1706814884138, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1268,7 +1314,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322096, + "lastObserved": 1706814884139, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1293,7 +1339,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628322096, + "lastObserved": 1706814884139, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json index ef7defc2ed589..266ecd965005e 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json @@ -22,7 +22,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250099, + "lastObserved": 1706812052262, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -38,7 +38,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250100, + "lastObserved": 1706812052263, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -54,7 +54,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250101, + "lastObserved": 1706812052264, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -72,7 +72,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250102, + "lastObserved": 1706812052265, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -97,7 +97,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250103, + "lastObserved": 1706812052266, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -113,7 +113,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250104, + "lastObserved": 1706812052267, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -133,15 +133,15 @@ "name": "personal_space", "description": "", "created": { - "time": 1706628249975 + "time": 1706812052156 }, "lastModified": { - "time": 1706628249975 + "time": 1706812052156 } } }, "systemMetadata": { - "lastObserved": 1706628250113, + "lastObserved": 1706812052274, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -157,7 +157,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250114, + "lastObserved": 1706812052275, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -173,7 +173,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250115, + "lastObserved": 1706812052276, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -191,7 +191,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250116, + "lastObserved": 1706812052277, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -207,43 +207,43 @@ } }, "systemMetadata": { - "lastObserved": 1706628250117, + "lastObserved": 1706812052278, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": { "platform": "qlik-sense", - "app": "b90c4d4e-0d07-4c24-9458-b17d1492660b", - "qri": "qri:app:sense://b90c4d4e-0d07-4c24-9458-b17d1492660b", + "app": "e09a68e7-18c9-461d-b957-043e0c045dcd", + "qri": "qri:app:sense://e09a68e7-18c9-461d-b957-043e0c045dcd", "usage": "ANALYTICS" }, - "name": "test_app_1", + "name": "IPL_Matches_2022", "description": "", "created": { - "time": 1704769768928 + "time": 1704803736545 }, "lastModified": { - "time": 1704771240100 + "time": 1705297020333 } } }, "systemMetadata": { - "lastObserved": 1706628250119, + "lastObserved": 1706812052280, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -252,14 +252,14 @@ } }, "systemMetadata": { - "lastObserved": 1706628250120, + "lastObserved": 1706812052281, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -268,14 +268,14 @@ } }, "systemMetadata": { - "lastObserved": 1706628250121, + "lastObserved": 1706812052281, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -286,14 +286,14 @@ } }, "systemMetadata": { - "lastObserved": 1706628250122, + "lastObserved": 1706812052282, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -311,44 +311,44 @@ } }, "systemMetadata": { - "lastObserved": 1706628250122, + "lastObserved": 1706812052283, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" + "container": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" } }, "systemMetadata": { - "lastObserved": 1706628250124, + "lastObserved": 1706812052284, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", - "urn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" } ] } }, "systemMetadata": { - "lastObserved": 1706628250124, + "lastObserved": 1706812052284, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -364,7 +364,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250126, + "lastObserved": 1706812052286, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -398,7 +398,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250127, + "lastObserved": 1706812052286, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -410,11 +410,11 @@ "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9" + "container": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" } }, "systemMetadata": { - "lastObserved": 1706628250130, + "lastObserved": 1706812052287, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -439,7 +439,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250131, + "lastObserved": 1706812052288, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -453,18 +453,18 @@ "json": { "path": [ { - "id": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", - "urn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" }, { - "id": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", - "urn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9" + "id": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "urn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" } ] } }, "systemMetadata": { - "lastObserved": 1706628250133, + "lastObserved": 1706812052289, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -480,7 +480,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250136, + "lastObserved": 1706812052290, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -508,14 +508,14 @@ } }, "systemMetadata": { - "lastObserved": 1706628250137, + "lastObserved": 1706812052291, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -524,19 +524,19 @@ } }, "systemMetadata": { - "lastObserved": 1706628250142, + "lastObserved": 1706812052293, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "659d8aef92df266ef3aa5a7c", + "schemaName": "harshal-playground-306419.test_dataset.test_table", "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { @@ -555,50 +555,14 @@ }, "fields": [ { - "fieldPath": "ID", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "City", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Date", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "Season", + "fieldPath": "name", "nullable": false, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.NullType": {} } }, - "nativeDataType": "INTEGER", + "nativeDataType": "", "recursive": false, "isPartOfKey": false } @@ -606,105 +570,95 @@ } }, "systemMetadata": { - "lastObserved": 1706628250144, + "lastObserved": 1706812052294, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { "json": { "customProperties": { - "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", - "spaceId": "659d0e41d1b0ecce6eebc9b1", - "type": "DELIMETED", - "size": "38168", - "rowCount": "74" - }, - "name": "IPL_Matches_2022.csv", - "qualifiedName": "IPL_Matches_2022.csv", - "description": "", - "created": { - "time": 1704803735527 - }, - "lastModified": { - "time": 1704804512453 + "dataconnectorid": "bb5be407-d3d3-4f19-858c-e71d593f09ae", + "dataconnectorName": "Google_BigQuery_harshal-playground-306419" }, + "name": "test_table", + "qualifiedName": "test_table", "tags": [] } }, "systemMetadata": { - "lastObserved": 1706628250149, + "lastObserved": 1706812052296, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + "container": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" } }, "systemMetadata": { - "lastObserved": 1706628250151, + "lastObserved": 1706812052297, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "subTypes", "aspect": { "json": { - "owners": [ - { - "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "typeNames": [ + "Qlik Dataset" + ] } }, "systemMetadata": { - "lastObserved": 1706628250153, + "lastObserved": 1706812052297, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "upstreamLineage", "aspect": { "json": { - "typeNames": [ - "Qlik Dataset" + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_dataset.test_table,PROD)", + "type": "COPY" + } ] } }, "systemMetadata": { - "lastObserved": 1706628250155, + "lastObserved": 1706812052298, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -713,98 +667,169 @@ { "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "urn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" } ] } }, "systemMetadata": { - "lastObserved": 1706628250156, + "lastObserved": 1706812052299, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "platform": "qlik-sense", - "app": "e09a68e7-18c9-461d-b957-043e0c045dcd", - "qri": "qri:app:sense://e09a68e7-18c9-461d-b957-043e0c045dcd", - "usage": "ANALYTICS" - }, - "name": "IPL_Matches_2022", - "description": "", - "created": { - "time": 1704803736545 - }, - "lastModified": { - "time": 1705297014684 - } + "removed": false } }, "systemMetadata": { - "lastObserved": 1706628250159, + "lastObserved": 1706812052301, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "schemaMetadata", "aspect": { "json": { - "removed": false + "schemaName": "659d8aef92df266ef3aa5a7c", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Season", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] } }, "systemMetadata": { - "lastObserved": 1706628250160, + "lastObserved": 1706812052302, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "datasetProperties", "aspect": { "json": { - "platform": "urn:li:dataPlatform:qlik-sense" + "customProperties": { + "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "type": "DELIMETED", + "size": "38168", + "rowCount": "74" + }, + "name": "IPL_Matches_2022.csv", + "qualifiedName": "IPL_Matches_2022.csv", + "description": "", + "created": { + "time": 1704803735527 + }, + "lastModified": { + "time": 1704804512453 + }, + "tags": [] } }, "systemMetadata": { - "lastObserved": 1706628250161, + "lastObserved": 1706812052304, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "container", "aspect": { "json": { - "typeNames": [ - "Qlik App" - ] + "container": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" } }, "systemMetadata": { - "lastObserved": 1706628250162, + "lastObserved": 1706812052305, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -822,30 +847,32 @@ } }, "systemMetadata": { - "lastObserved": 1706628250163, + "lastObserved": 1706812052306, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + "typeNames": [ + "Qlik Dataset" + ] } }, "systemMetadata": { - "lastObserved": 1706628250164, + "lastObserved": 1706812052307, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -859,7 +886,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250165, + "lastObserved": 1706812052307, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -875,7 +902,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250168, + "lastObserved": 1706812052309, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -921,7 +948,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250170, + "lastObserved": 1706812052310, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -953,7 +980,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250173, + "lastObserved": 1706812052312, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -969,7 +996,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250175, + "lastObserved": 1706812052313, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -994,7 +1021,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250177, + "lastObserved": 1706812052314, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1012,7 +1039,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250179, + "lastObserved": 1706812052315, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1033,7 +1060,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250180, + "lastObserved": 1706812052315, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1049,7 +1076,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250182, + "lastObserved": 1706812052316, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1065,28 +1092,28 @@ } }, "systemMetadata": { - "lastObserved": 1706628250183, + "lastObserved": 1706812052317, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", + "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ { - "id": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", - "urn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" } ] } }, "systemMetadata": { - "lastObserved": 1706628250185, + "lastObserved": 1706812052317, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1100,25 +1127,25 @@ "json": { "path": [ { - "id": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", - "urn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" }, { - "id": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9", - "urn": "urn:li:container:152b86a9f687c1c85a9b7d55481757e9" + "id": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "urn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" } ] } }, "systemMetadata": { - "lastObserved": 1706628250188, + "lastObserved": 1706812052318, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1127,19 +1154,23 @@ { "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "urn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" } ] } }, "systemMetadata": { - "lastObserved": 1706628250189, + "lastObserved": 1706812052319, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1153,7 +1184,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250191, + "lastObserved": 1706812052320, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1174,7 +1205,7 @@ } }, "systemMetadata": { - "lastObserved": 1706628250192, + "lastObserved": 1706812052320, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py index 7341c588fc3b3..c53b0515528f2 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py +++ b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py @@ -58,101 +58,54 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "json": { "data": [ { - "name": "test_app_1", + "name": "test_app", + "spaceId": "659d0e41d1b0ecce6eebc9b1", "thumbnailId": "", "resourceAttributes": { "_resourcetype": "app", - "createdDate": "2024-01-09T08:39:28.928Z", + "createdDate": "2024-01-25T17:52:24.922Z", "description": "", "dynamicColor": "", "hasSectionAccess": False, - "id": "b90c4d4e-0d07-4c24-9458-b17d1492660b", + "id": "f0714ca7-7093-49e4-8b58-47bb38563647", "isDirectQueryMode": False, - "lastReloadTime": "2024-01-09T09:03:59.227Z", - "modifiedDate": "2024-01-09T09:04:00.100Z", - "name": "test_app_1", + "lastReloadTime": "2024-01-25T17:56:00.902Z", + "modifiedDate": "2024-01-25T17:56:02.045Z", + "name": "test_app", "originAppId": "", "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", "ownerId": "657b5abe656297cec3d8b205", "publishTime": "", "published": False, - "spaceId": "", + "spaceId": "659d0e41d1b0ecce6eebc9b1", "thumbnail": "", "usage": "ANALYTICS", }, "resourceCustomAttributes": None, - "resourceUpdatedAt": "2024-01-09T09:04:00Z", + "resourceUpdatedAt": "2024-01-25T17:56:02Z", "resourceType": "app", - "resourceId": "b90c4d4e-0d07-4c24-9458-b17d1492660b", - "resourceCreatedAt": "2024-01-09T08:39:28Z", - "id": "659d064133e7d51dbcbb5911", - "createdAt": "2024-01-09T08:39:29Z", - "updatedAt": "2024-01-09T09:04:00Z", + "resourceId": "f0714ca7-7093-49e4-8b58-47bb38563647", + "resourceCreatedAt": "2024-01-25T17:52:24Z", + "id": "65b29fd9435c545ed042a807", + "createdAt": "2024-01-25T17:52:25Z", + "updatedAt": "2024-01-25T17:56:02Z", "creatorId": "657b5abe656297cec3d8b205", "updaterId": "657b5abe656297cec3d8b205", "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", "isFavorited": False, - "links": { - "self": { - "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/659d064133e7d51dbcbb5911" - }, - "collections": { - "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/659d064133e7d51dbcbb5911/collections" - }, - "open": { - "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/b90c4d4e-0d07-4c24-9458-b17d1492660b" - }, - }, - "actions": [ - "change_owner", - "change_space", - "create", - "create_session_app", - "delete", - "delete_share", - "duplicate", - "export", - "export_reduced", - "exportappdata", - "import", - "read", - "reload", - "source", - "update", - ], - "collectionIds": ["659d0c6dad4c8e1b1e1f6ebe"], + "collectionIds": [], "meta": { "isFavorited": False, - "actions": [ - "change_owner", - "change_space", - "create", - "create_session_app", - "delete", - "delete_share", - "duplicate", - "export", - "export_reduced", - "exportappdata", - "import", - "read", - "reload", - "source", - "update", - ], - "tags": [], - "collections": [ - { - "id": "659d0c6dad4c8e1b1e1f6ebe", - "name": "test_collection", - } + "tags": [ + {"id": "659ce561640a2affcf0d629f", "name": "test_tag"} ], + "collections": [], }, "ownerId": "657b5abe656297cec3d8b205", - "resourceReloadEndTime": "2024-01-09T09:03:59Z", + "resourceReloadEndTime": "2024-01-25T17:56:00Z", "resourceReloadStatus": "ok", - "resourceSize": {"appFile": 196608, "appMemory": 625}, - "itemViews": {"total": 10, "trend": 0.5, "unique": 2}, + "resourceSize": {"appFile": 2057, "appMemory": 78}, + "itemViews": {"total": 3, "trend": 0.3, "unique": 1}, }, { "name": "IPL_Matches_2022.csv", @@ -182,33 +135,8 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "updaterId": "657b5abe656297cec3d8b205", "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", "isFavorited": False, - "links": { - "self": { - "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/659d8aef12794f37026cb262" - }, - "collections": { - "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/659d8aef12794f37026cb262/collections" - }, - }, - "actions": [ - "create", - "delete", - "list", - "profile", - "read", - "update", - ], "collectionIds": [], "meta": { - "isFavorited": False, - "actions": [ - "create", - "delete", - "list", - "profile", - "read", - "update", - ], "tags": [], "collections": [], }, @@ -218,99 +146,6 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "resourceSize": {"appFile": 0, "appMemory": 0}, "itemViews": {}, }, - { - "name": "IPL_Matches_2022", - "spaceId": "659d0e41d1b0ecce6eebc9b1", - "thumbnailId": "", - "resourceAttributes": { - "_resourcetype": "app", - "createdDate": "2024-01-09T18:05:36.545Z", - "description": "", - "dynamicColor": "", - "hasSectionAccess": False, - "id": "e09a68e7-18c9-461d-b957-043e0c045dcd", - "isDirectQueryMode": False, - "lastReloadTime": "2024-01-15T11:06:53.070Z", - "modifiedDate": "2024-01-15T11:06:54.684Z", - "name": "IPL_Matches_2022", - "originAppId": "", - "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", - "ownerId": "657b5abe656297cec3d8b205", - "publishTime": "", - "published": False, - "spaceId": "659d0e41d1b0ecce6eebc9b1", - "thumbnail": "", - "usage": "ANALYTICS", - }, - "resourceCustomAttributes": None, - "resourceUpdatedAt": "2024-01-15T11:06:54Z", - "resourceType": "app", - "resourceId": "e09a68e7-18c9-461d-b957-043e0c045dcd", - "resourceCreatedAt": "2024-01-09T18:05:36Z", - "id": "659d8af1ef4eadeead3ec0ec", - "createdAt": "2024-01-09T18:05:37Z", - "updatedAt": "2024-01-15T11:06:54Z", - "creatorId": "657b5abe656297cec3d8b205", - "updaterId": "657b5abe656297cec3d8b205", - "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", - "isFavorited": False, - "links": { - "self": { - "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/659d8af1ef4eadeead3ec0ec" - }, - "collections": { - "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/659d8af1ef4eadeead3ec0ec/collections" - }, - "open": { - "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/e09a68e7-18c9-461d-b957-043e0c045dcd" - }, - }, - "actions": [ - "change_owner", - "change_space", - "create", - "delete", - "delete_share", - "duplicate", - "export", - "export_reduced", - "exportappdata", - "import", - "read", - "reload", - "share", - "source", - "update", - ], - "collectionIds": [], - "meta": { - "isFavorited": False, - "actions": [ - "change_owner", - "change_space", - "create", - "delete", - "delete_share", - "duplicate", - "export", - "export_reduced", - "exportappdata", - "import", - "read", - "reload", - "share", - "source", - "update", - ], - "tags": [], - "collections": [], - }, - "ownerId": "657b5abe656297cec3d8b205", - "resourceReloadEndTime": "2024-01-15T11:06:53Z", - "resourceReloadStatus": "ok", - "resourceSize": {"appFile": 17634, "appMemory": 36315}, - "itemViews": {"total": 4, "trend": 0.5, "unique": 2}, - }, { "name": "test_tabl", "resourceAttributes": { @@ -338,33 +173,8 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "updaterId": "657b5abe656297cec3d8b205", "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", "isFavorited": False, - "links": { - "self": { - "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/65a137c8d5a03b02d359624a" - }, - "collections": { - "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items/65a137c8d5a03b02d359624a/collections" - }, - }, - "actions": [ - "create", - "delete", - "list", - "profile", - "read", - "update", - ], "collectionIds": [], "meta": { - "isFavorited": False, - "actions": [ - "create", - "delete", - "list", - "profile", - "read", - "update", - ], "tags": [], "collections": [], }, @@ -372,12 +182,6 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "resourceReloadEndTime": "", "resourceReloadStatus": "", "resourceSize": {"appFile": 0, "appMemory": 0}, - "itemViews": { - "total": 2, - "trend": 0.3, - "unique": 1, - "usedBy": 1, - }, }, ], }, @@ -595,94 +399,69 @@ def mock_websocket_response(*args, **kwargs): "id": 1, "handle": -1, "method": "OpenDoc", - "params": {"qDocName": "b90c4d4e-0d07-4c24-9458-b17d1492660b"}, + "params": {"qDocName": "f0714ca7-7093-49e4-8b58-47bb38563647"}, }: return { - "jsonrpc": "2.0", - "id": 1, - "result": { - "qReturn": { - "qType": "Doc", - "qHandle": 1, - "qGenericId": "b90c4d4e-0d07-4c24-9458-b17d1492660b", - } - }, - "change": [1], + "qReturn": { + "qType": "Doc", + "qHandle": 1, + "qGenericId": "f0714ca7-7093-49e4-8b58-47bb38563647", + } } elif request == { "jsonrpc": "2.0", "id": 2, "handle": 1, - "method": "GetObjects", - "params": {"qOptions": {"qTypes": ["sheet"]}}, + "method": "GetAppLayout", + "params": {}, }: return { - "jsonrpc": "2.0", - "id": 2, - "result": { - "qList": [ - { - "qInfo": { - "qId": "f4f57386-263a-4ec9-b40c-abcd2467f423", - "qType": "sheet", - }, - "qMeta": { - "title": "New ds sheet", - "description": "", - "_resourcetype": "app.object", - "_objecttype": "sheet", - "id": "f4f57386-263a-4ec9-b40c-abcd2467f423", - "approved": False, - "published": False, - "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", - "ownerId": "657b5abe656297cec3d8b205", - "createdDate": "2024-01-15T11:01:49.704Z", - "modifiedDate": "2024-01-29T12:23:46.868Z", - "publishTime": True, - "privileges": [ - "read", - "update", - "delete", - "publish", - "change_owner", - ], - }, - "qData": {}, - }, - ] - }, + "qLayout": { + "qTitle": "IPL_Matches_2022", + "qFileName": "e09a68e7-18c9-461d-b957-043e0c045dcd", + "qLastReloadTime": "2024-01-15T11:06:53.070Z", + "qHasScript": True, + "qStateNames": [], + "qMeta": {}, + "qHasData": True, + "qThumbnail": {}, + "qUnsupportedFeatures": [], + "qUsage": "ANALYTICS", + "encrypted": True, + "id": "e09a68e7-18c9-461d-b957-043e0c045dcd", + "published": False, + "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "ownerId": "657b5abe656297cec3d8b205", + "createdDate": "2024-01-09T18:05:36.545Z", + "modifiedDate": "2024-01-15T11:07:00.333Z", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "hassectionaccess": False, + "_resourcetype": "app", + "privileges": [ + "read", + "update", + "delete", + "reload", + "export", + "duplicate", + "change_owner", + "change_space", + "export_reduced", + "source", + "exportappdata", + ], + } } elif request == { "jsonrpc": "2.0", "id": 3, "handle": 1, - "method": "GetObject", - "params": {"qId": "f4f57386-263a-4ec9-b40c-abcd2467f423"}, - }: - return { - "jsonrpc": "2.0", - "id": 3, - "result": { - "qReturn": { - "qType": "GenericObject", - "qHandle": 2, - "qGenericType": "sheet", - "qGenericId": "f4f57386-263a-4ec9-b40c-abcd2467f423", - } - }, - } - elif request == { - "jsonrpc": "2.0", - "id": 4, - "handle": 2, - "method": "GetLayout", - "params": {}, + "method": "GetObjects", + "params": {"qOptions": {"qTypes": ["sheet"]}}, }: return { - "jsonrpc": "2.0", - "id": 4, - "result": { - "qLayout": { + "qList": [ + { "qInfo": { "qId": "f4f57386-263a-4ec9-b40c-abcd2467f423", "qType": "sheet", @@ -708,52 +487,194 @@ def mock_websocket_response(*args, **kwargs): "change_owner", ], }, - "qSelectionInfo": {}, - "rank": 0, - "thumbnail": {"qStaticContentUrl": {}}, - "columns": 24, - "rows": 12, - "cells": [ + "qData": {}, + }, + ] + } + elif request == { + "jsonrpc": "2.0", + "id": 4, + "handle": 1, + "method": "GetObject", + "params": {"qId": "f4f57386-263a-4ec9-b40c-abcd2467f423"}, + }: + return { + "qReturn": { + "qType": "GenericObject", + "qHandle": 2, + "qGenericType": "sheet", + "qGenericId": "f4f57386-263a-4ec9-b40c-abcd2467f423", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 5, + "handle": 2, + "method": "GetLayout", + "params": {}, + }: + return { + "qLayout": { + "qInfo": { + "qId": "f4f57386-263a-4ec9-b40c-abcd2467f423", + "qType": "sheet", + }, + "qMeta": { + "title": "New ds sheet", + "description": "", + "_resourcetype": "app.object", + "_objecttype": "sheet", + "id": "f4f57386-263a-4ec9-b40c-abcd2467f423", + "approved": False, + "published": False, + "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "ownerId": "657b5abe656297cec3d8b205", + "createdDate": "2024-01-15T11:01:49.704Z", + "modifiedDate": "2024-01-29T12:23:46.868Z", + "publishTime": True, + "privileges": [ + "read", + "update", + "delete", + "publish", + "change_owner", + ], + }, + "qSelectionInfo": {}, + "rank": 0, + "thumbnail": {"qStaticContentUrl": {}}, + "columns": 24, + "rows": 12, + "qChildList": { + "qItems": [ { - "bounds": {"x": 0, "y": 0, "width": 100, "height": 100}, - "col": 0, - "colspan": 24, - "name": "QYUUb", - "row": 0, - "rowspan": 12, - "smartGrid": { - "rowIdx": 0, - "itemIdx": 0, - "item": {"width": 1}, - }, - "type": "barchart", + "qInfo": {"qId": "QYUUb", "qType": "barchart"}, + "qMeta": {"privileges": ["read", "update", "delete"]}, + "qData": {"title": ""}, } - ], - "qChildList": { - "qItems": [ + ] + }, + "customRowBase": 12, + "gridResolution": "small", + "layoutOptions": {"mobileLayout": "LIST", "extendable": False}, + "gridMode": "simpleEdit", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 6, + "handle": 1, + "method": "GetObject", + "params": ["LoadModel"], + }: + return { + "qReturn": { + "qType": "GenericObject", + "qHandle": 3, + "qGenericType": "LoadModel", + "qGenericId": "LoadModel", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 7, + "handle": 3, + "method": "GetLayout", + "params": {}, + }: + return { + "qLayout": { + "qInfo": {"qId": "LoadModel", "qType": "LoadModel"}, + "tables": [ + { + "dataconnectorName": "Google_BigQuery_harshal-playground-306419", + "dataconnectorPrefix": "test_space:", + "boxType": "blackbox", + "databaseName": "", + "ownerName": "", + "tableName": "test_table", + "tableAlias": "test_table", + "loadProperties": { + "filterInfo": {"filterClause": "", "filterType": 1} + }, + "key": "Google_BigQuery_harshal-playground-306419:::test_table", + "fields": [ { - "qInfo": {"qId": "QYUUb", "qType": "barchart"}, - "qMeta": {"privileges": ["read", "update", "delete"]}, - "qData": {"title": ""}, + "alias": "name", + "name": "name", + "selected": True, + "checked": True, + "id": "dsd.test_table.name", } - ] - }, - "customRowBase": 12, - "gridResolution": "small", - "layoutOptions": {"mobileLayout": "LIST", "extendable": False}, - "gridMode": "simpleEdit", - } - }, + ], + "connectionInfo": { + "name": "Google_BigQuery_harshal-playground-306419", + "displayName": "Google_BigQuery_harshal-playground-306419", + "id": "bb5be407-d3d3-4f19-858c-e71d593f09ae", + "type": { + "provider": "QvOdbcConnectorPackage.exe", + "type": "custom", + "name": "QvOdbcConnectorPackage", + "displayName": "Qlik® ODBC Connector Package", + "isStandardConnector": False, + "isIframeCompatible": True, + "needsConnect": True, + "connectDialog": "/customdata/64/QvOdbcConnectorPackage/web/standalone/connect-dialog.html?locale=en-US", + "selectDialog": "/customdata/64/QvOdbcConnectorPackage/web/standalone/select-dialog.html?locale=en-US", + "selectAddData": "/customdata/64/QvOdbcConnectorPackage/web/standalone/select-adddata.html?locale=en-US", + "credentialsDialog": "/customdata/64/QvOdbcConnectorPackage/web/standalone/credentials-dialog.html?locale=en-US", + "update": "/customdata/64/QvOdbcConnectorPackage/web/standalone/loadModelUpdate.js", + "architecture": {"text": "Common.undefinedbit"}, + "connectorMain": "QvOdbcConnectorPackage.webroot/connector-main-iframe", + }, + "typeName": "QvOdbcConnectorPackage.exe", + "privileges": [ + "change_owner", + "change_space", + "delete", + "list", + "read", + "update", + ], + "sourceConnectorID": "gbq", + "dataconnectorPrefix": "test_space:", + "isInAppSpace": True, + "space": "659d0e41d1b0ecce6eebc9b1", + "connectionString": 'CUSTOM CONNECT TO "provider=QvOdbcConnectorPackage.exe;driver=gbq;OAuthMechanism=0;SupportOldClient=true;Catalog_Old=harshal-playground-306419;separateCredentials=false;Catalog=harshal-playground-306419;Min_TLS=1.2;SQLDialect=1;RowsFetchedPerBlock=16384;DefaultStringColumnLength=65535;AllowLargeResults=false;EnableHTAPI=false;HTAPI_MinResultsSize=1000;HTAPI_MinActivationRatio=3;allowNonSelectQueries=false;QueryTimeout=30;Timeout=300;useBulkReader=true;bulkFetchSize=50;rowBatchSize=1;bulkFetchColumnMode=true;maxStringLength=4096;logSQLStatements=false;"', + "hasEditableSeparatedCredentials": False, + "canDelete": True, + "connectorImagePath": "https://iq37k6byr9lgam8.us.qlikcloud.com/customdata/64/QvOdbcConnectorPackage/web/gbq-square.png", + "connectorDisplayName": "Google BigQuery", + "dbInfo": {}, + }, + "id": "dsd.test_table", + "tableGroupId": "", + "connectorProperties": { + "tableQualifiers": [ + "harshal-playground-306419", + "test_dataset", + ], + }, + "selectStatement": "SELECT name\nFROM `harshal-playground-306419`.`test_dataset`.`test_table`;", + "caching": {"enabled": True, "type": "qvd"}, + } + ], + "schemaVersion": 2.1, + } } + else: + import pdb + + pdb.set_trace() + return {} @pytest.fixture(scope="module") def mock_websocket_send_request(): - with patch( - "datahub.ingestion.source.qlik_sense.qlik_api.QlikAPI._websocket_send_request" + "datahub.ingestion.source.qlik_sense.qlik_api.WebsocketConnection._send_request" ) as mock_websocket_send_request, patch( - "datahub.ingestion.source.qlik_sense.qlik_api.create_connection" + "datahub.ingestion.source.qlik_sense.websocket_connection.create_connection" ): mock_websocket_send_request.side_effect = mock_websocket_response yield mock_websocket_send_request @@ -763,12 +684,7 @@ def default_config(): return { "tenant_hostname": "iq37k6byr9lgam8.us.qlikcloud.com", "api_key": "qlik-api-key", - "space_pattern": { - "allow": [ - "test_space", - ] - }, - "extract_personal_entity": True, + "space_pattern": {"allow": ["test_space", "personal_space"]}, } @@ -831,6 +747,12 @@ def test_platform_instance_ingest( "config": { **default_config(), "platform_instance": "qlik_sense_platform", + "app_dataset_source_to_platform_instance": { + "harshal-playground-306419.test_dataset.test_table": { + "platform_instance": "google-cloud", + "env": "DEV", + } + }, }, }, "sink": { @@ -843,7 +765,7 @@ def test_platform_instance_ingest( ) pipeline.run() pipeline.raise_from_status() - golden_file = "golden_test_platform_instace_ingest.json" + golden_file = "golden_test_platform_instance_ingest.json" mce_helpers.check_golden_file( pytestconfig, From 924ab3f2a2be7490045080faef0e88496df8e311 Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Sun, 4 Feb 2024 17:39:27 +0530 Subject: [PATCH 07/15] Address review comments --- .../sources/qlik-sense/qlik-sense_recipe.yml | 8 +- .../ingestion/source/qlik_sense/config.py | 21 +-- .../ingestion/source/qlik_sense/qlik_api.py | 16 +++ .../ingestion/source/qlik_sense/qlik_sense.py | 35 +++-- .../golden_test_platform_instance_ingest.json | 128 +++++++++--------- .../golden_test_qlik_sense_ingest.json | 120 ++++++++-------- .../integration/qlik_sense/test_qlik_sense.py | 36 ++++- 7 files changed, 203 insertions(+), 161 deletions(-) diff --git a/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml index afcbcc9ed804f..315e3d1de3f7f 100644 --- a/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml +++ b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml @@ -14,10 +14,10 @@ source: ingest_owner: true - # Optional -- This mapping is optional and only required to configure platform-instance for Qlik app dataset upstream source table - # A mapping of the Qlik app dataset upstream source to platform instance. Use ..
as key. - # app_dataset_source_to_platform_instance: - # ..
: + # Optional -- This mapping is optional and only required to configure platform-instance for Qlik app dataset upstream source tables + # A mapping of the Qlik app dataset upstream tables from data connection to platform instance. Use 'data_connection_name' as key. + # data_connection_to_platform_instance: + # data_connection_name: # platform_instance: cloud_instance # env: DEV diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py index 735d11769963a..d3303b04a8cec 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py @@ -21,20 +21,6 @@ QLIK_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" -class WebSocketRequest: - """ - Web socket send request dict - """ - - OPEN_DOC = { - "jsonrpc": "2.0", - "id": 1, - "handle": -1, - "method": "OpenDoc", - "params": {"qDocName": "f0714ca7-7093-49e4-8b58-47bb38563647"}, - } - - class Constant: """ keys used in qlik plugin @@ -122,8 +108,9 @@ class QlikSourceConfig( default=True, description="Ingest Owner from source. This will override Owner info entered from UI", ) - # Qlik app dataset upstream source to platform instance mapping - app_dataset_source_to_platform_instance: Dict[str, PlatformDetail] = pydantic.Field( + # Qlik app dataset upstream tables from data connection to platform instance mapping + data_connection_to_platform_instance: Dict[str, PlatformDetail] = pydantic.Field( default={}, - description="A mapping of the Qlik app dataset upstream source to platform instance. Use ..
as key.", + description="A mapping of the Qlik app dataset upstream tables from data connection to platform instance." + "Use 'data_connection_name' as key.", ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py index 267d2d424fa69..3bc64d692766d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py @@ -24,6 +24,7 @@ class QlikAPI: def __init__(self, config: QlikSourceConfig) -> None: self.spaces: Dict = {} + self.users: Dict = {} self.config = config self.session = requests.Session() self.session.headers.update( @@ -74,6 +75,21 @@ def _get_dataset(self, dataset_id: str) -> Optional[QlikDataset]: ) return None + def get_user_name(self, user_id: str) -> Optional[str]: + try: + if user_id in self.users: + # To avoid fetching same user details again + return self.users[user_id] + else: + response = self.session.get(f"{self.rest_api_url}/users/{user_id}") + response.raise_for_status() + user_name = response.json()[Constant.NAME] + self.users[user_id] = user_name + return user_name + except Exception: + self._log_http_error(message=f"Unable to fetch user with id {user_id}") + return None + def _get_sheet( self, websocket_connection: WebsocketConnection, diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py index d1fc2b88a7477..1cfd329c35b73 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py @@ -171,14 +171,17 @@ def _gen_space_workunit(self, space: Space) -> Iterable[MetadataWorkUnit]: """ Map Qlik space to Datahub container """ + owner_username: Optional[str] = None + if space.ownerId: + owner_username = self.qlik_api.get_user_name(space.ownerId) yield from gen_containers( container_key=self._gen_space_key(space.id), name=space.name, description=space.description, sub_types=[BIContainerSubTypes.QLIK_SPACE], extra_properties={Constant.TYPE: str(space.type)}, - owner_urn=builder.make_user_urn(space.ownerId) - if self.config.ingest_owner and space.ownerId + owner_urn=builder.make_user_urn(owner_username) + if self.config.ingest_owner and owner_username else None, created=int(space.createdAt.timestamp() * 1000), last_modified=int(space.updatedAt.timestamp() * 1000), @@ -190,12 +193,12 @@ def _gen_entity_status_aspect(self, entity_urn: str) -> MetadataWorkUnit: ).as_workunit() def _gen_entity_owner_aspect( - self, entity_urn: str, owner_id: str + self, entity_urn: str, user_name: str ) -> MetadataWorkUnit: aspect = OwnershipClass( owners=[ OwnerClass( - owner=builder.make_user_urn(owner_id), + owner=builder.make_user_urn(user_name), type=OwnershipTypeClass.DATAOWNER, ) ] @@ -281,8 +284,9 @@ def _gen_sheets_workunit( if dpi_aspect: yield dpi_aspect - if self.config.ingest_owner: - yield self._gen_entity_owner_aspect(dashboard_urn, sheet.ownerId) + owner_username = self.qlik_api.get_user_name(sheet.ownerId) + if self.config.ingest_owner and owner_username: + yield self._gen_entity_owner_aspect(dashboard_urn, owner_username) yield from self._gen_charts_workunit(sheet.charts) @@ -292,8 +296,8 @@ def _gen_app_dataset_upstream_lineage( dataset_identifier = self._get_app_dataset_identifier(dataset) dataset_urn = self._gen_qlik_dataset_urn(dataset_identifier) upstream_dataset_platform_detail = ( - self.config.app_dataset_source_to_platform_instance.get( - dataset_identifier, + self.config.data_connection_to_platform_instance.get( + dataset.dataconnectorName, PlatformDetail( env=self.config.env, platform_instance=self.config.platform_instance ), @@ -339,7 +343,9 @@ def _gen_app_dataset_properties(self, dataset: QlikAppDataset) -> MetadataWorkUn ).as_workunit() def _get_app_dataset_identifier(self, dataset: QlikAppDataset) -> str: - return f"{dataset.databaseName}.{dataset.schemaName}.{dataset.tableName}" + return ( + f"{dataset.databaseName}.{dataset.schemaName}.{dataset.tableName}".lower() + ) def _gen_app_dataset_workunit( self, datasets: List[QlikAppDataset], app_id: str @@ -375,6 +381,7 @@ def _gen_app_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: """ Map Qlik App to Datahub container """ + owner_username = self.qlik_api.get_user_name(app.ownerId) yield from gen_containers( container_key=self._gen_app_key(app.id), name=app.qTitle, @@ -382,8 +389,8 @@ def _gen_app_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: sub_types=[BIContainerSubTypes.QLIK_APP], parent_container_key=self._gen_space_key(app.spaceId), extra_properties={Constant.QRI: app.qri, Constant.USAGE: app.qUsage}, - owner_urn=builder.make_user_urn(app.ownerId) - if self.config.ingest_owner + owner_urn=builder.make_user_urn(owner_username) + if self.config.ingest_owner and owner_username else None, created=int(app.createdAt.timestamp() * 1000), last_modified=int(app.updatedAt.timestamp() * 1000), @@ -427,7 +434,6 @@ def _gen_schema_fields( if field.dataType else NullType() ), - # NOTE: nativeDataType will not be in sync with older connector nativeDataType=field.dataType if field.dataType else "", nullable=field.nullable, isPartOfKey=field.primaryKey, @@ -496,8 +502,9 @@ def _gen_dataset_workunit(self, dataset: QlikDataset) -> Iterable[MetadataWorkUn if dpi_aspect: yield dpi_aspect - if self.config.ingest_owner: - yield self._gen_entity_owner_aspect(dataset_urn, dataset.ownerId) + owner_username = self.qlik_api.get_user_name(dataset.ownerId) + if self.config.ingest_owner and owner_username: + yield self._gen_entity_owner_aspect(dataset_urn, owner_username) yield MetadataChangeProposalWrapper( entityUrn=dataset_urn, diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json index d3864749f0179..92415127d414e 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json @@ -23,7 +23,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884085, + "lastObserved": 1707047363253, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -39,7 +39,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884086, + "lastObserved": 1707047363254, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -56,7 +56,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884087, + "lastObserved": 1707047363255, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -74,7 +74,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884087, + "lastObserved": 1707047363256, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -88,7 +88,7 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "owner": "urn:li:corpuser:Shubham jagtap", "type": "DATAOWNER" } ], @@ -99,7 +99,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884088, + "lastObserved": 1707047363256, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -120,7 +120,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884089, + "lastObserved": 1707047363257, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -141,15 +141,15 @@ "name": "personal_space", "description": "", "created": { - "time": 1706814883987 + "time": 1707047363158 }, "lastModified": { - "time": 1706814883987 + "time": 1707047363158 } } }, "systemMetadata": { - "lastObserved": 1706814884096, + "lastObserved": 1707047363264, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -165,7 +165,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884097, + "lastObserved": 1707047363265, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -182,7 +182,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884098, + "lastObserved": 1707047363266, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -200,7 +200,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884098, + "lastObserved": 1707047363266, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -221,7 +221,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884099, + "lastObserved": 1707047363267, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -251,7 +251,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884100, + "lastObserved": 1707047363268, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -267,7 +267,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884101, + "lastObserved": 1707047363269, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -284,7 +284,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884102, + "lastObserved": 1707047363269, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -302,7 +302,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884102, + "lastObserved": 1707047363270, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -316,7 +316,7 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "owner": "urn:li:corpuser:Shubham jagtap", "type": "DATAOWNER" } ], @@ -327,7 +327,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884103, + "lastObserved": 1707047363271, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -343,7 +343,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884104, + "lastObserved": 1707047363272, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -368,7 +368,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884104, + "lastObserved": 1707047363272, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -384,7 +384,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884106, + "lastObserved": 1707047363274, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -418,7 +418,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884106, + "lastObserved": 1707047363274, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -434,7 +434,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884107, + "lastObserved": 1707047363275, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -451,7 +451,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884108, + "lastObserved": 1707047363276, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -465,7 +465,7 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "owner": "urn:li:corpuser:Shubham jagtap", "type": "DATAOWNER" } ], @@ -476,7 +476,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884109, + "lastObserved": 1707047363276, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -505,7 +505,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884109, + "lastObserved": 1707047363277, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -521,7 +521,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884110, + "lastObserved": 1707047363278, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -549,7 +549,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884111, + "lastObserved": 1707047363279, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -565,7 +565,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884113, + "lastObserved": 1707047363281, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -611,7 +611,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884114, + "lastObserved": 1707047363282, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -633,7 +633,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884115, + "lastObserved": 1707047363283, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -649,7 +649,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884116, + "lastObserved": 1707047363284, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -666,7 +666,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884117, + "lastObserved": 1707047363284, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -684,7 +684,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884117, + "lastObserved": 1707047363285, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -709,7 +709,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884118, + "lastObserved": 1707047363286, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -738,7 +738,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884119, + "lastObserved": 1707047363287, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -754,7 +754,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884120, + "lastObserved": 1707047363288, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -836,7 +836,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884121, + "lastObserved": 1707047363289, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -868,7 +868,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884123, + "lastObserved": 1707047363291, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -884,7 +884,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884124, + "lastObserved": 1707047363292, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -901,7 +901,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884125, + "lastObserved": 1707047363293, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -915,7 +915,7 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "owner": "urn:li:corpuser:Shubham jagtap", "type": "DATAOWNER" } ], @@ -926,7 +926,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884126, + "lastObserved": 1707047363293, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -944,7 +944,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884126, + "lastObserved": 1707047363294, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -969,7 +969,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884127, + "lastObserved": 1707047363295, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -985,7 +985,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884128, + "lastObserved": 1707047363296, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1031,7 +1031,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884129, + "lastObserved": 1707047363297, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1063,7 +1063,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884130, + "lastObserved": 1707047363298, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1079,7 +1079,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884131, + "lastObserved": 1707047363299, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1096,7 +1096,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884132, + "lastObserved": 1707047363300, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1110,7 +1110,7 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "owner": "urn:li:corpuser:Shubham jagtap", "type": "DATAOWNER" } ], @@ -1121,7 +1121,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884133, + "lastObserved": 1707047363300, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1139,7 +1139,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884134, + "lastObserved": 1707047363301, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1164,7 +1164,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884134, + "lastObserved": 1707047363302, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1185,7 +1185,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884135, + "lastObserved": 1707047363303, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1206,7 +1206,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884136, + "lastObserved": 1707047363303, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1231,7 +1231,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884136, + "lastObserved": 1707047363304, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1260,7 +1260,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884137, + "lastObserved": 1707047363305, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1289,7 +1289,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884138, + "lastObserved": 1707047363306, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1314,7 +1314,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884139, + "lastObserved": 1707047363306, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1339,7 +1339,7 @@ } }, "systemMetadata": { - "lastObserved": 1706814884139, + "lastObserved": 1707047363307, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json index 266ecd965005e..2dd62c5e56940 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json @@ -22,7 +22,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052262, + "lastObserved": 1707047264838, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -38,7 +38,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052263, + "lastObserved": 1707047264839, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -54,7 +54,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052264, + "lastObserved": 1707047264840, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -72,7 +72,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052265, + "lastObserved": 1707047264840, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -86,7 +86,7 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "owner": "urn:li:corpuser:Shubham jagtap", "type": "DATAOWNER" } ], @@ -97,7 +97,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052266, + "lastObserved": 1707047264841, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -113,7 +113,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052267, + "lastObserved": 1707047264842, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -133,15 +133,15 @@ "name": "personal_space", "description": "", "created": { - "time": 1706812052156 + "time": 1707047264739 }, "lastModified": { - "time": 1706812052156 + "time": 1707047264739 } } }, "systemMetadata": { - "lastObserved": 1706812052274, + "lastObserved": 1707047264849, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -157,7 +157,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052275, + "lastObserved": 1707047264850, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -173,7 +173,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052276, + "lastObserved": 1707047264850, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -191,7 +191,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052277, + "lastObserved": 1707047264851, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -207,7 +207,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052278, + "lastObserved": 1707047264852, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -236,7 +236,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052280, + "lastObserved": 1707047264853, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -252,7 +252,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052281, + "lastObserved": 1707047264854, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -268,7 +268,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052281, + "lastObserved": 1707047264854, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -286,7 +286,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052282, + "lastObserved": 1707047264855, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -300,7 +300,7 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "owner": "urn:li:corpuser:Shubham jagtap", "type": "DATAOWNER" } ], @@ -311,7 +311,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052283, + "lastObserved": 1707047264856, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -327,7 +327,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052284, + "lastObserved": 1707047264856, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -348,7 +348,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052284, + "lastObserved": 1707047264857, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -364,7 +364,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052286, + "lastObserved": 1707047264858, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -398,7 +398,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052286, + "lastObserved": 1707047264859, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -414,7 +414,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052287, + "lastObserved": 1707047264860, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -428,7 +428,7 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "owner": "urn:li:corpuser:Shubham jagtap", "type": "DATAOWNER" } ], @@ -439,7 +439,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052288, + "lastObserved": 1707047264861, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -464,7 +464,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052289, + "lastObserved": 1707047264861, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -480,7 +480,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052290, + "lastObserved": 1707047264862, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -508,7 +508,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052291, + "lastObserved": 1707047264863, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -524,7 +524,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052293, + "lastObserved": 1707047264865, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -570,7 +570,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052294, + "lastObserved": 1707047264866, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -592,7 +592,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052296, + "lastObserved": 1707047264867, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -608,7 +608,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052297, + "lastObserved": 1707047264868, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -626,7 +626,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052297, + "lastObserved": 1707047264869, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -651,7 +651,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052298, + "lastObserved": 1707047264869, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -676,7 +676,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052299, + "lastObserved": 1707047264870, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -692,7 +692,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052301, + "lastObserved": 1707047264872, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -774,7 +774,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052302, + "lastObserved": 1707047264872, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -806,7 +806,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052304, + "lastObserved": 1707047264874, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -822,7 +822,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052305, + "lastObserved": 1707047264875, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -836,7 +836,7 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "owner": "urn:li:corpuser:Shubham jagtap", "type": "DATAOWNER" } ], @@ -847,7 +847,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052306, + "lastObserved": 1707047264876, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -865,7 +865,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052307, + "lastObserved": 1707047264877, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -886,7 +886,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052307, + "lastObserved": 1707047264878, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -902,7 +902,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052309, + "lastObserved": 1707047264880, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -948,7 +948,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052310, + "lastObserved": 1707047264881, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -980,7 +980,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052312, + "lastObserved": 1707047264882, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -996,7 +996,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052313, + "lastObserved": 1707047264883, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1010,7 +1010,7 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:657b5abe656297cec3d8b205", + "owner": "urn:li:corpuser:Shubham jagtap", "type": "DATAOWNER" } ], @@ -1021,7 +1021,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052314, + "lastObserved": 1707047264884, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1039,7 +1039,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052315, + "lastObserved": 1707047264885, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1060,7 +1060,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052315, + "lastObserved": 1707047264886, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1076,7 +1076,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052316, + "lastObserved": 1707047264886, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1092,7 +1092,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052317, + "lastObserved": 1707047264887, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1113,7 +1113,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052317, + "lastObserved": 1707047264888, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1138,7 +1138,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052318, + "lastObserved": 1707047264888, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1163,7 +1163,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052319, + "lastObserved": 1707047264889, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1184,7 +1184,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052320, + "lastObserved": 1707047264890, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1205,7 +1205,7 @@ } }, "systemMetadata": { - "lastObserved": 1706812052320, + "lastObserved": 1707047264891, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py index c53b0515528f2..a0bbdc4348e1b 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py +++ b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py @@ -186,6 +186,38 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: ], }, }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/users/657b5abe656297cec3d8b205": { + "method": "GET", + "status_code": 200, + "json": { + "id": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "status": "active", + "subject": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "name": "Shubham jagtap", + "email": "Shubham.Jagtap@gslab.com", + "roles": [ + "TenantAdmin", + "AnalyticsAdmin", + "AuditAdmin", + "DataAdmin", + "Developer", + "SharedSpaceCreator", + "PrivateAnalyticsContentCreator", + "DataServicesContributor", + ], + "groups": [], + "createdAt": "2023-12-14T19:42:54.417Z", + "lastUpdatedAt": "2024-01-25T06:28:22.629Z", + "created": "2023-12-14T19:42:54.417Z", + "lastUpdated": "2024-01-25T06:28:22.629Z", + "links": { + "self": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/users/657b5abe656297cec3d8b205" + } + }, + }, + }, "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/data-sets/659d8aef92df266ef3aa5a7c": { "method": "GET", "status_code": 200, @@ -747,8 +779,8 @@ def test_platform_instance_ingest( "config": { **default_config(), "platform_instance": "qlik_sense_platform", - "app_dataset_source_to_platform_instance": { - "harshal-playground-306419.test_dataset.test_table": { + "data_connection_to_platform_instance": { + "Google_BigQuery_harshal-playground-306419": { "platform_instance": "google-cloud", "env": "DEV", } From 6c48007eeab9a861ac0d319271d00dfc5dae8d37 Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Thu, 8 Feb 2024 00:13:26 +0530 Subject: [PATCH 08/15] Address review comments --- .../ingestion/source/qlik_sense/config.py | 9 + .../source/qlik_sense/data_classes.py | 49 +- .../ingestion/source/qlik_sense/qlik_api.py | 138 +++-- .../ingestion/source/qlik_sense/qlik_sense.py | 162 +++--- .../golden_test_platform_instance_ingest.json | 473 +++++++++++++----- .../golden_test_qlik_sense_ingest.json | 434 ++++++++++++---- .../integration/qlik_sense/test_qlik_sense.py | 311 +++++++++++- 7 files changed, 1230 insertions(+), 346 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py index d3303b04a8cec..1bcd89dfcb491 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py @@ -31,6 +31,8 @@ class Constant: ID = "id" NAME = "name" TYPE = "type" + ITEMID = "itemId" + DATASETTYPE = "datasetType" CREATEDAT = "createdAt" UPDATEDAT = "updatedAt" SECUREQRI = "secureQri" @@ -52,10 +54,17 @@ class Constant: MODIFIEDDATE = "modifiedDate" RESOURCEID = "resourceId" DATASETSCHEMA = "datasetSchema" + GRAPH = "graph" + NODES = "nodes" + RESOURCES = "resources" + LINEAGE = "lineage" + TABLELABEL = "tableLabel" + TABLEQRI = "tableQRI" # Websocket response key constants QID = "qId" RESULT = "result" QRETURN = "qReturn" + QTYPE = "qType" QHANDLE = "qHandle" QLAYOUT = "qLayout" QMETA = "qMeta" diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py index d3caf80f2bb94..a7c6f76464b63 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py @@ -61,6 +61,12 @@ class SpaceType(Enum): DATA = "data" +# Qlik table box type +class BoxType(Enum): + LOADFILE = "load-file" # Table extracted from dataset + BLACKBOX = "blackbox" # Table extracted from data connection + + PERSONAL_SPACE_DICT = { "id": Constant.PERSONAL_SPACE_ID, "name": Constant.PERSONAL_SPACE_NAME, @@ -118,6 +124,7 @@ class QlikDataset(Item): type: str size: int rowCount: int + itemId: str datasetSchema: List[SchemaField] @root_validator(pre=True) @@ -140,9 +147,26 @@ def update_values(cls, values: Dict) -> Dict: return values +class AxisProperty(BaseModel): + Title: str = Field(alias="qFallbackTitle") + Min: str = Field(alias="qMin") + Max: str = Field(alias="qMax") + + class Chart(BaseModel): qId: str - qType: str + visualization: str + title: str + subtitle: str + qDimension: List[AxisProperty] + qMeasure: List[AxisProperty] + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.QID] = values[Constant.QINFO][Constant.QID] + values["qDimension"] = values["qHyperCube"]["qDimensionInfo"] + values["qMeasure"] = values["qHyperCube"]["qMeasureInfo"] + return values class Sheet(BaseModel): @@ -165,28 +189,33 @@ def update_values(cls, values: Dict) -> Dict: return values -class QlikAppDataset(BaseModel): +class QlikTable(BaseModel): tableName: str - schemaName: str databaseName: str + type: BoxType = Field(alias="boxType") tableAlias: str dataconnectorid: str dataconnectorName: str dataconnectorPlatform: str + spaceId: str datasetSchema: List[SchemaField] = Field(alias="fields") + tableQri: Optional[str] = None + schemaName: Optional[str] = None @root_validator(pre=True) def update_values(cls, values: Dict) -> Dict: - values[Constant.DATABASENAME] = values[Constant.CONNECTORPROPERTIES][ - Constant.TABLEQUALIFIERS - ][0] - values[Constant.SCHEMANAME] = values[Constant.CONNECTORPROPERTIES][ - Constant.TABLEQUALIFIERS - ][1] + if values["boxType"] == BoxType.BLACKBOX.value: + values[Constant.DATABASENAME] = values[Constant.CONNECTORPROPERTIES][ + Constant.TABLEQUALIFIERS + ][0] + values[Constant.SCHEMANAME] = values[Constant.CONNECTORPROPERTIES][ + Constant.TABLEQUALIFIERS + ][1] values[Constant.DATACONNECTORID] = values[Constant.CONNECTIONINFO][Constant.ID] values[Constant.DATACONNECTORPLATFORM] = values[Constant.CONNECTIONINFO][ Constant.SOURCECONNECTORID ] + values[Constant.SPACEID] = values[Constant.CONNECTIONINFO]["space"] return values @@ -195,7 +224,7 @@ class App(Item): qri: str qUsage: str sheets: List[Sheet] = [] - datasets: List[QlikAppDataset] = [] + tables: List[QlikTable] = [] @root_validator(pre=True) def update_values(cls, values: Dict) -> Dict: diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py index 3bc64d692766d..d5d559144ef16 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py @@ -1,6 +1,7 @@ import logging import sys from typing import Any, Dict, List, Optional +from urllib.parse import quote import requests @@ -10,8 +11,8 @@ App, Chart, Item, - QlikAppDataset, QlikDataset, + QlikTable, Sheet, Space, ) @@ -60,18 +61,20 @@ def get_spaces(self) -> List[Space]: self.spaces[PERSONAL_SPACE_DICT[Constant.ID]] = PERSONAL_SPACE_DICT[ Constant.NAME ] - except Exception: - self._log_http_error(message="Unable to fetch spaces") + except Exception as e: + self._log_http_error(message=f"Unable to fetch spaces. Exception: {e}") return spaces - def _get_dataset(self, dataset_id: str) -> Optional[QlikDataset]: + def _get_dataset(self, dataset_id: str, item_id: str) -> Optional[QlikDataset]: try: response = self.session.get(f"{self.rest_api_url}/data-sets/{dataset_id}") response.raise_for_status() - return QlikDataset.parse_obj(response.json()) - except Exception: + response_dict = response.json() + response_dict[Constant.ITEMID] = item_id + return QlikDataset.parse_obj(response_dict) + except Exception as e: self._log_http_error( - message=f"Unable to fetch dataset with id {dataset_id}" + message=f"Unable to fetch dataset with id {dataset_id}. Exception: {e}" ) return None @@ -86,8 +89,28 @@ def get_user_name(self, user_id: str) -> Optional[str]: user_name = response.json()[Constant.NAME] self.users[user_id] = user_name return user_name - except Exception: - self._log_http_error(message=f"Unable to fetch user with id {user_id}") + except Exception as e: + self._log_http_error( + message=f"Unable to fetch user with id {user_id}. Exception: {e}" + ) + return None + + def _get_chart( + self, + websocket_connection: WebsocketConnection, + chart_id: str, + sheet_id: str, + ) -> Optional[Chart]: + try: + websocket_connection.websocket_send_request( + method="GetChild", params={"qId": chart_id} + ) + response = websocket_connection.websocket_send_request(method="GetLayout") + return Chart.parse_obj(response[Constant.QLAYOUT]) + except Exception as e: + self._log_http_error( + message=f"Unable to fetch chart {chart_id} of sheet {sheet_id}. Exception: {e}" + ) return None def _get_sheet( @@ -102,33 +125,83 @@ def _get_sheet( response = websocket_connection.websocket_send_request(method="GetLayout") sheet_dict = response[Constant.QLAYOUT] sheet = Sheet.parse_obj(sheet_dict[Constant.QMETA]) + i = 1 for chart_dict in sheet_dict[Constant.QCHILDLIST][Constant.QITEMS]: - sheet.charts.append(Chart.parse_obj(chart_dict[Constant.QINFO])) + chart = self._get_chart( + websocket_connection, + chart_dict[Constant.QINFO][Constant.QID], + sheet_id, + ) + if chart: + if not chart.title: + chart.title = f"Object {i}" + i += 1 + sheet.charts.append(chart) + websocket_connection.handle.pop() return sheet - except Exception: - self._log_http_error(message=f"Unable to fetch sheet with id {sheet_id}") + except Exception as e: + self._log_http_error( + message=f"Unable to fetch sheet with id {sheet_id}. Exception: {e}" + ) return None - def _get_app_used_datasets( + def _add_qri_of_tables(self, tables: List[QlikTable], app_id: str) -> None: + table_qri_dict: Dict[str, str] = {} + app_qri = quote(f"qri:app:sense://{app_id}", safe="") + try: + response = self.session.get( + f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/actions/expand?node={app_qri}&level=TABLE" + ) + response.raise_for_status() + for table_node_qri in response.json()[Constant.GRAPH][Constant.NODES]: + table_node_qri = quote(table_node_qri, safe="") + response = self.session.get( + f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/actions/expand?node={table_node_qri}&level=FIELD" + ) + response.raise_for_status() + field_node_qri = list( + response.json()[Constant.GRAPH][Constant.NODES].keys() + )[0] + response = self.session.post( + f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/overview", + json=[field_node_qri], + ) + response.raise_for_status() + for each_lineage in response.json()[Constant.RESOURCES][0][ + Constant.LINEAGE + ]: + table_qri_dict[each_lineage[Constant.TABLELABEL]] = each_lineage[ + Constant.TABLEQRI + ] + for table in tables: + if table.tableName in table_qri_dict: + table.tableQri = table_qri_dict[table.tableName] + except Exception as e: + self._log_http_error( + message=f"Unable to add QRI for tables of app {app_id}. Exception: {e}" + ) + + def _get_app_used_tables( self, websocket_connection: WebsocketConnection, app_id: str - ) -> List[QlikAppDataset]: - datasets: List[QlikAppDataset] = [] + ) -> List[QlikTable]: + tables: List[QlikTable] = [] try: - websocket_connection.websocket_send_request( + response = websocket_connection.websocket_send_request( method="GetObject", params=["LoadModel"], ) + if not response[Constant.QRETURN][Constant.QTYPE]: + return [] response = websocket_connection.websocket_send_request(method="GetLayout") for table_dict in response[Constant.QLAYOUT][Constant.TABLES]: - # Condition to Add connection based table only - if table_dict["boxType"] == "blackbox": - datasets.append(QlikAppDataset.parse_obj(table_dict)) + tables.append(QlikTable.parse_obj(table_dict)) websocket_connection.handle.pop() - except Exception: + self._add_qri_of_tables(tables, app_id) + except Exception as e: self._log_http_error( - message=f"Unable to fetch app used datasets for app {app_id}" + message=f"Unable to fetch tables used by app {app_id}. Exception: {e}" ) - return datasets + return tables def _get_app_sheets( self, websocket_connection: WebsocketConnection, app_id: str @@ -151,8 +224,10 @@ def _get_app_sheets( if sheet: sheets.append(sheet) websocket_connection.handle.pop() - except Exception: - self._log_http_error(message=f"Unable to fetch sheets for app {app_id}") + except Exception as e: + self._log_http_error( + message=f"Unable to fetch sheets for app {app_id}. Exception: {e}" + ) return sheets def _get_app(self, app_id: str) -> Optional[App]: @@ -169,11 +244,13 @@ def _get_app(self, app_id: str) -> Optional[App]: ) app = App.parse_obj(response[Constant.QLAYOUT]) app.sheets = self._get_app_sheets(websocket_connection, app_id) - app.datasets = self._get_app_used_datasets(websocket_connection, app_id) + app.tables = self._get_app_used_tables(websocket_connection, app_id) websocket_connection.close_websocket() return app - except Exception: - self._log_http_error(message=f"Unable to fetch app with id {app_id}") + except Exception as e: + self._log_http_error( + message=f"Unable to fetch app with id {app_id}. Exception: {e}" + ) return None def get_items(self) -> List[Item]: @@ -196,11 +273,12 @@ def get_items(self) -> List[Item]: items.append(app) elif resource_type == Constant.DATASET: dataset = self._get_dataset( - dataset_id=item[Constant.RESOURCEID] + dataset_id=item[Constant.RESOURCEID], + item_id=item[Constant.ID], ) if dataset: items.append(dataset) - except Exception: - self._log_http_error(message="Unable to fetch items") + except Exception as e: + self._log_http_error(message=f"Unable to fetch items. Exception: {e}") return items diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py index 1cfd329c35b73..3cdc5c846dd79 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py @@ -37,9 +37,10 @@ KNOWN_DATA_PLATFORM_MAPPING, App, AppKey, + BoxType, Chart, - QlikAppDataset, QlikDataset, + QlikTable, SchemaField as QlikDatasetSchemaField, Sheet, Space, @@ -183,6 +184,7 @@ def _gen_space_workunit(self, space: Space) -> Iterable[MetadataWorkUnit]: owner_urn=builder.make_user_urn(owner_username) if self.config.ingest_owner and owner_username else None, + external_url=f"https://{self.config.tenant_hostname}/catalog?space_filter={space.id}", created=int(space.createdAt.timestamp() * 1000), last_modified=int(space.updatedAt.timestamp() * 1000), ) @@ -215,7 +217,9 @@ def _gen_dashboard_urn(self, dashboard_identifier: str) -> str: name=dashboard_identifier, ) - def _gen_dashboard_info_workunit(self, sheet: Sheet) -> MetadataWorkUnit: + def _gen_dashboard_info_workunit( + self, sheet: Sheet, app_id: str + ) -> MetadataWorkUnit: dashboard_urn = self._gen_dashboard_urn(sheet.id) custom_properties: Dict[str, str] = {"chartCount": str(len(sheet.charts))} dashboard_info_cls = DashboardInfoClass( @@ -234,6 +238,7 @@ def _gen_dashboard_info_workunit(self, sheet: Sheet) -> MetadataWorkUnit: lastModified=self._get_audit_stamp(sheet.updatedAt, sheet.ownerId), ), customProperties=custom_properties, + dashboardUrl=f"https://{self.config.tenant_hostname}/sense/app/{app_id}/sheet/{sheet.id}/state/analysis", ) return MetadataChangeProposalWrapper( entityUrn=dashboard_urn, aspect=dashboard_info_cls @@ -252,17 +257,23 @@ def _gen_charts_workunit(self, charts: List[Chart]) -> Iterable[MetadataWorkUnit yield self._gen_entity_status_aspect(chart_urn) + custom_properties = { + "Dimension": str(chart.qDimension), + "Measure": str(chart.qMeasure), + } + yield MetadataChangeProposalWrapper( entityUrn=chart_urn, aspect=ChartInfoClass( - title=chart.qId, - description=chart.qType, + title=chart.title, + description=chart.visualization, lastModified=ChangeAuditStampsClass(), + customProperties=custom_properties, ), ).as_workunit() def _gen_sheets_workunit( - self, sheets: List[Sheet], space_id: str, app_id: str + self, sheets: List[Sheet], app_id: str ) -> Iterable[MetadataWorkUnit]: """ Map Qlik Sheet to Datahub dashboard @@ -272,7 +283,7 @@ def _gen_sheets_workunit( yield self._gen_entity_status_aspect(dashboard_urn) - yield self._gen_dashboard_info_workunit(sheet) + yield self._gen_dashboard_info_workunit(sheet, app_id) yield from add_entity_to_container( container_key=self._gen_app_key(app_id), @@ -290,75 +301,83 @@ def _gen_sheets_workunit( yield from self._gen_charts_workunit(sheet.charts) - def _gen_app_dataset_upstream_lineage( - self, dataset: QlikAppDataset - ) -> MetadataWorkUnit: - dataset_identifier = self._get_app_dataset_identifier(dataset) - dataset_urn = self._gen_qlik_dataset_urn(dataset_identifier) - upstream_dataset_platform_detail = ( - self.config.data_connection_to_platform_instance.get( - dataset.dataconnectorName, - PlatformDetail( - env=self.config.env, platform_instance=self.config.platform_instance + def _gen_app_table_upstream_lineage( + self, dataset_urn: str, table: QlikTable + ) -> Optional[MetadataWorkUnit]: + upstream_dataset_urn: Optional[str] = None + if table.type == BoxType.BLACKBOX: + upstream_dataset_platform_detail = ( + self.config.data_connection_to_platform_instance.get( + table.dataconnectorName, + PlatformDetail( + env=self.config.env, + platform_instance=self.config.platform_instance, + ), + ) + ) + upstream_dataset_urn = builder.make_dataset_urn_with_platform_instance( + name=f"{table.databaseName}.{table.schemaName}.{table.tableName}", + platform=KNOWN_DATA_PLATFORM_MAPPING.get( + table.dataconnectorPlatform, table.dataconnectorPlatform ), + env=upstream_dataset_platform_detail.env, + platform_instance=upstream_dataset_platform_detail.platform_instance, ) - ) - - upstream_dataset_urn = builder.make_dataset_urn_with_platform_instance( - name=dataset_identifier, - platform=KNOWN_DATA_PLATFORM_MAPPING.get( - dataset.dataconnectorPlatform, dataset.dataconnectorPlatform - ), - env=upstream_dataset_platform_detail.env, - platform_instance=upstream_dataset_platform_detail.platform_instance, - ) - - return MetadataChangeProposalWrapper( - entityUrn=dataset_urn, - aspect=UpstreamLineage( - upstreams=[ - Upstream(dataset=upstream_dataset_urn, type=DatasetLineageType.COPY) - ] - ), - ).as_workunit() - - def _gen_app_dataset_properties(self, dataset: QlikAppDataset) -> MetadataWorkUnit: - dataset_urn = self._gen_qlik_dataset_urn( - self._get_app_dataset_identifier(dataset) - ) + elif table.type == BoxType.LOADFILE: + upstream_dataset_urn = self._gen_qlik_dataset_urn( + f"{table.spaceId}.{table.databaseName}" + ) + if upstream_dataset_urn: + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=UpstreamLineage( + upstreams=[ + Upstream( + dataset=upstream_dataset_urn, type=DatasetLineageType.COPY + ) + ] + ), + ).as_workunit() + else: + return None + def _gen_app_table_properties( + self, dataset_urn: str, table: QlikTable + ) -> MetadataWorkUnit: dataset_properties = DatasetProperties( - name=dataset.tableName, - qualifiedName=dataset.tableAlias, + name=table.tableName, + qualifiedName=table.tableAlias, ) dataset_properties.customProperties.update( { - Constant.DATACONNECTORID: dataset.dataconnectorid, - Constant.DATACONNECTORNAME: dataset.dataconnectorName, + Constant.TYPE: "Qlik Table", + Constant.DATACONNECTORID: table.dataconnectorid, + Constant.DATACONNECTORNAME: table.dataconnectorName, } ) - return MetadataChangeProposalWrapper( entityUrn=dataset_urn, aspect=dataset_properties ).as_workunit() - def _get_app_dataset_identifier(self, dataset: QlikAppDataset) -> str: - return ( - f"{dataset.databaseName}.{dataset.schemaName}.{dataset.tableName}".lower() - ) + def _get_app_table_identifier(self, table: QlikTable) -> Optional[str]: + if table.tableQri: + return table.tableQri.lower() + return None - def _gen_app_dataset_workunit( - self, datasets: List[QlikAppDataset], app_id: str + def _gen_app_tables_workunit( + self, tables: List[QlikTable], app_id: str ) -> Iterable[MetadataWorkUnit]: - for dataset in datasets: - dataset_identifier = self._get_app_dataset_identifier(dataset) - dataset_urn = self._gen_qlik_dataset_urn(dataset_identifier) + for table in tables: + table_identifier = self._get_app_table_identifier(table) + if not table_identifier: + continue + dataset_urn = self._gen_qlik_dataset_urn(table_identifier) yield self._gen_entity_status_aspect(dataset_urn) - yield self._gen_schema_metadata(dataset_identifier, dataset.datasetSchema) + yield self._gen_schema_metadata(table_identifier, table.datasetSchema) - yield self._gen_app_dataset_properties(dataset) + yield self._gen_app_table_properties(dataset_urn, table) yield from add_entity_to_container( container_key=self._gen_app_key(app_id), @@ -375,7 +394,11 @@ def _gen_app_dataset_workunit( aspect=SubTypes(typeNames=[DatasetSubTypes.QLIK_DATASET]), ).as_workunit() - yield self._gen_app_dataset_upstream_lineage(dataset) + upstream_lineage_workunit = self._gen_app_table_upstream_lineage( + dataset_urn, table + ) + if upstream_lineage_workunit: + yield upstream_lineage_workunit def _gen_app_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: """ @@ -392,11 +415,12 @@ def _gen_app_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: owner_urn=builder.make_user_urn(owner_username) if self.config.ingest_owner and owner_username else None, + external_url=f"https://{self.config.tenant_hostname}/sense/app/{app.id}/overview", created=int(app.createdAt.timestamp() * 1000), last_modified=int(app.updatedAt.timestamp() * 1000), ) - yield from self._gen_sheets_workunit(app.sheets, app.spaceId, app.id) - yield from self._gen_app_dataset_workunit(app.datasets, app.id) + yield from self._gen_sheets_workunit(app.sheets, app.id) + yield from self._gen_app_tables_workunit(app.tables, app.id) def _gen_qlik_dataset_urn(self, dataset_identifier: str) -> str: return builder.make_dataset_urn_with_platform_instance( @@ -459,13 +483,14 @@ def _gen_schema_metadata( entityUrn=dataset_urn, aspect=schema_metadata ).as_workunit() - def _gen_dataset_properties(self, dataset: QlikDataset) -> MetadataWorkUnit: - dataset_urn = self._gen_qlik_dataset_urn(dataset.id) - + def _gen_dataset_properties( + self, dataset_urn: str, dataset: QlikDataset + ) -> MetadataWorkUnit: dataset_properties = DatasetProperties( name=dataset.name, description=dataset.description, qualifiedName=dataset.name, + externalUrl=f"https://{self.config.tenant_hostname}/dataset/{dataset.itemId}", created=TimeStamp(time=int(dataset.createdAt.timestamp() * 1000)), lastModified=TimeStamp(time=int(dataset.updatedAt.timestamp() * 1000)), ) @@ -473,7 +498,8 @@ def _gen_dataset_properties(self, dataset: QlikDataset) -> MetadataWorkUnit: { Constant.QRI: dataset.secureQri, Constant.SPACEID: dataset.spaceId, - Constant.TYPE: dataset.type, + Constant.TYPE: "Qlik Dataset", + Constant.DATASETTYPE: dataset.type, Constant.SIZE: str(dataset.size), Constant.ROWCOUNT: str(dataset.rowCount), } @@ -483,14 +509,18 @@ def _gen_dataset_properties(self, dataset: QlikDataset) -> MetadataWorkUnit: entityUrn=dataset_urn, aspect=dataset_properties ).as_workunit() + def _get_qlik_dataset_identifier(self, dataset: QlikDataset) -> str: + return f"{dataset.spaceId}.{dataset.name}".lower() + def _gen_dataset_workunit(self, dataset: QlikDataset) -> Iterable[MetadataWorkUnit]: - dataset_urn = self._gen_qlik_dataset_urn(dataset.id) + dataset_identifier = self._get_qlik_dataset_identifier(dataset) + dataset_urn = self._gen_qlik_dataset_urn(dataset_identifier) yield self._gen_entity_status_aspect(dataset_urn) - yield self._gen_schema_metadata(dataset.id, dataset.datasetSchema) + yield self._gen_schema_metadata(dataset_identifier, dataset.datasetSchema) - yield self._gen_dataset_properties(dataset) + yield self._gen_dataset_properties(dataset_urn, dataset) yield from add_entity_to_container( container_key=self._gen_space_key(dataset.spaceId), diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json index 92415127d414e..7cfe225dddbd7 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json @@ -12,6 +12,7 @@ "space": "659d0e41d1b0ecce6eebc9b1", "type": "SpaceType.SHARED" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=659d0e41d1b0ecce6eebc9b1", "name": "test_space", "description": "", "created": { @@ -23,7 +24,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363253, + "lastObserved": 1707327546102, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -39,7 +40,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363254, + "lastObserved": 1707327546103, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -56,7 +57,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363255, + "lastObserved": 1707327546104, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -74,7 +75,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363256, + "lastObserved": 1707327546105, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -99,7 +100,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363256, + "lastObserved": 1707327546106, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -120,7 +121,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363257, + "lastObserved": 1707327546106, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -138,18 +139,19 @@ "space": "personal-space-id", "type": "SpaceType.PERSONAL" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=personal-space-id", "name": "personal_space", "description": "", "created": { - "time": 1707047363158 + "time": 1707327546001 }, "lastModified": { - "time": 1707047363158 + "time": 1707327546001 } } }, "systemMetadata": { - "lastObserved": 1707047363264, + "lastObserved": 1707327546121, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -165,7 +167,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363265, + "lastObserved": 1707327546122, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -182,7 +184,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363266, + "lastObserved": 1707327546123, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -200,7 +202,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363266, + "lastObserved": 1707327546124, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -221,14 +223,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363267, + "lastObserved": 1707327546124, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { @@ -236,10 +238,11 @@ "customProperties": { "platform": "qlik-sense", "instance": "qlik_sense_platform", - "app": "e09a68e7-18c9-461d-b957-043e0c045dcd", - "qri": "qri:app:sense://e09a68e7-18c9-461d-b957-043e0c045dcd", + "app": "f0714ca7-7093-49e4-8b58-47bb38563647", + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647", "usage": "ANALYTICS" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/overview", "name": "IPL_Matches_2022", "description": "", "created": { @@ -251,14 +254,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363268, + "lastObserved": 1707327546126, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -267,14 +270,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363269, + "lastObserved": 1707327546127, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -284,14 +287,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363269, + "lastObserved": 1707327546127, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -302,14 +305,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363270, + "lastObserved": 1707327546128, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -327,14 +330,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363271, + "lastObserved": 1707327546129, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -343,14 +346,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363272, + "lastObserved": 1707327546130, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -368,7 +371,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363272, + "lastObserved": 1707327546130, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -384,7 +387,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363274, + "lastObserved": 1707327546132, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -414,11 +417,12 @@ "time": 1706511226868, "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" } - } + }, + "dashboardUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/sheet/f4f57386-263a-4ec9-b40c-abcd2467f423/state/analysis" } }, "systemMetadata": { - "lastObserved": 1707047363274, + "lastObserved": 1707327546132, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -430,11 +434,11 @@ "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" + "container": "urn:li:container:784b26286a16989c6329d372ccc2f97e" } }, "systemMetadata": { - "lastObserved": 1707047363275, + "lastObserved": 1707327546133, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -451,7 +455,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363276, + "lastObserved": 1707327546134, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -476,7 +480,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363276, + "lastObserved": 1707327546135, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -498,14 +502,14 @@ "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" }, { - "id": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", - "urn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" } ] } }, "systemMetadata": { - "lastObserved": 1707047363277, + "lastObserved": 1707327546136, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -521,7 +525,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363278, + "lastObserved": 1707327546137, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -533,9 +537,12 @@ "aspectName": "chartInfo", "aspect": { "json": { - "customProperties": {}, - "title": "QYUUb", - "description": "barchart", + "customProperties": { + "Dimension": "[AxisProperty(Title='City', Min='NaN', Max='NaN')]", + "Measure": "[AxisProperty(Title='Sum(Date)', Min='89411', Max='2144260')]" + }, + "title": "Test_chart", + "description": "scatterplot", "lastModified": { "created": { "time": 0, @@ -549,14 +556,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363279, + "lastObserved": 1707327546138, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -565,19 +572,19 @@ } }, "systemMetadata": { - "lastObserved": 1707047363281, + "lastObserved": 1707327546139, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "harshal-playground-306419.test_dataset.test_table", + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje", "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { @@ -611,19 +618,20 @@ } }, "systemMetadata": { - "lastObserved": 1707047363282, + "lastObserved": 1707327546140, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { "json": { "customProperties": { + "type": "Qlik Table", "dataconnectorid": "bb5be407-d3d3-4f19-858c-e71d593f09ae", "dataconnectorName": "Google_BigQuery_harshal-playground-306419" }, @@ -633,30 +641,30 @@ } }, "systemMetadata": { - "lastObserved": 1707047363283, + "lastObserved": 1707327546142, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" + "container": "urn:li:container:784b26286a16989c6329d372ccc2f97e" } }, "systemMetadata": { - "lastObserved": 1707047363284, + "lastObserved": 1707327546142, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -666,14 +674,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363284, + "lastObserved": 1707327546143, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -684,14 +692,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363285, + "lastObserved": 1707327546144, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -709,14 +717,216 @@ } }, "systemMetadata": { - "lastObserved": 1707047363286, + "lastObserved": 1707327546144, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546145, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546147, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546147, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "type": "Qlik Table", + "dataconnectorid": "87d7bc7e-77d8-40dc-a251-3a35ec107b4e", + "dataconnectorName": "DataFiles" + }, + "name": "IPL_Matches_2022", + "qualifiedName": "IPL_Matches_2022", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327546149, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + }, + "systemMetadata": { + "lastObserved": 1707327546150, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546151, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546151, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.IPL_Matches_2022.csv,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546152, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -731,21 +941,21 @@ "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" }, { - "id": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", - "urn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" } ] } }, "systemMetadata": { - "lastObserved": 1707047363287, + "lastObserved": 1707327546153, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -754,19 +964,19 @@ } }, "systemMetadata": { - "lastObserved": 1707047363288, + "lastObserved": 1707327546155, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "659d8aef92df266ef3aa5a7c", + "schemaName": "659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv", "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { @@ -836,14 +1046,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363289, + "lastObserved": 1707327546155, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -851,10 +1061,12 @@ "customProperties": { "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", "spaceId": "659d0e41d1b0ecce6eebc9b1", - "type": "DELIMETED", + "type": "Qlik Dataset", + "datasetType": "DELIMETED", "size": "38168", "rowCount": "74" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/659d8aef12794f37026cb262", "name": "IPL_Matches_2022.csv", "qualifiedName": "IPL_Matches_2022.csv", "description": "", @@ -868,14 +1080,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363291, + "lastObserved": 1707327546158, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -884,14 +1096,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363292, + "lastObserved": 1707327546159, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -901,14 +1113,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363293, + "lastObserved": 1707327546160, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -926,14 +1138,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363293, + "lastObserved": 1707327546161, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -944,14 +1156,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363294, + "lastObserved": 1707327546161, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -969,14 +1181,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363295, + "lastObserved": 1707327546162, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -985,19 +1197,19 @@ } }, "systemMetadata": { - "lastObserved": 1707047363296, + "lastObserved": 1707327546163, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "65a137c849f82a37c625151b", + "schemaName": "personal-space-id.test_tabl", "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { @@ -1031,14 +1243,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363297, + "lastObserved": 1707327546164, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -1046,10 +1258,12 @@ "customProperties": { "qri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", "spaceId": "personal-space-id", - "type": "CONNECTION_BASED_DATASET", + "type": "Qlik Dataset", + "datasetType": "CONNECTION_BASED_DATASET", "size": "0", "rowCount": "1" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/65a137c8d5a03b02d359624a", "name": "test_tabl", "qualifiedName": "test_tabl", "description": "", @@ -1063,14 +1277,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363298, + "lastObserved": 1707327546166, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -1079,14 +1293,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363299, + "lastObserved": 1707327546167, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -1096,14 +1310,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363300, + "lastObserved": 1707327546167, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -1121,14 +1335,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363300, + "lastObserved": 1707327546168, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1139,14 +1353,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363301, + "lastObserved": 1707327546169, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1164,7 +1378,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363302, + "lastObserved": 1707327546170, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1185,7 +1399,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363303, + "lastObserved": 1707327546170, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1206,14 +1420,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363303, + "lastObserved": 1707327546171, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1231,7 +1445,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363304, + "lastObserved": 1707327546172, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1253,21 +1467,50 @@ "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" }, { - "id": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", - "urn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546172, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" } ] } }, "systemMetadata": { - "lastObserved": 1707047363305, + "lastObserved": 1707327546173, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1282,21 +1525,21 @@ "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" }, { - "id": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b", - "urn": "urn:li:container:cd1646eaf9b0a9d63f0cc1aaf36aeb3b" + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" } ] } }, "systemMetadata": { - "lastObserved": 1707047363306, + "lastObserved": 1707327546174, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1314,14 +1557,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047363306, + "lastObserved": 1707327546175, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1339,7 +1582,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047363307, + "lastObserved": 1707327546176, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json index 2dd62c5e56940..32af5d2191de4 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json @@ -11,6 +11,7 @@ "space": "659d0e41d1b0ecce6eebc9b1", "type": "SpaceType.SHARED" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=659d0e41d1b0ecce6eebc9b1", "name": "test_space", "description": "", "created": { @@ -22,7 +23,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264838, + "lastObserved": 1707327442983, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -38,7 +39,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264839, + "lastObserved": 1707327442985, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -54,7 +55,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264840, + "lastObserved": 1707327442985, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -72,7 +73,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264840, + "lastObserved": 1707327442986, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -97,7 +98,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264841, + "lastObserved": 1707327442987, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -113,7 +114,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264842, + "lastObserved": 1707327442988, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -130,18 +131,19 @@ "space": "personal-space-id", "type": "SpaceType.PERSONAL" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=personal-space-id", "name": "personal_space", "description": "", "created": { - "time": 1707047264739 + "time": 1707327442833 }, "lastModified": { - "time": 1707047264739 + "time": 1707327442833 } } }, "systemMetadata": { - "lastObserved": 1707047264849, + "lastObserved": 1707327443007, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -157,7 +159,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264850, + "lastObserved": 1707327443008, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -173,7 +175,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264850, + "lastObserved": 1707327443010, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -191,7 +193,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264851, + "lastObserved": 1707327443010, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -207,24 +209,25 @@ } }, "systemMetadata": { - "lastObserved": 1707047264852, + "lastObserved": 1707327443011, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { "customProperties": { "platform": "qlik-sense", - "app": "e09a68e7-18c9-461d-b957-043e0c045dcd", - "qri": "qri:app:sense://e09a68e7-18c9-461d-b957-043e0c045dcd", + "app": "f0714ca7-7093-49e4-8b58-47bb38563647", + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647", "usage": "ANALYTICS" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/overview", "name": "IPL_Matches_2022", "description": "", "created": { @@ -236,14 +239,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264853, + "lastObserved": 1707327443012, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -252,14 +255,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264854, + "lastObserved": 1707327443014, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -268,14 +271,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264854, + "lastObserved": 1707327443014, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -286,14 +289,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264855, + "lastObserved": 1707327443015, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -311,14 +314,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264856, + "lastObserved": 1707327443016, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -327,14 +330,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264856, + "lastObserved": 1707327443017, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -348,7 +351,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264857, + "lastObserved": 1707327443018, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -364,7 +367,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264858, + "lastObserved": 1707327443019, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -394,11 +397,12 @@ "time": 1706511226868, "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" } - } + }, + "dashboardUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/sheet/f4f57386-263a-4ec9-b40c-abcd2467f423/state/analysis" } }, "systemMetadata": { - "lastObserved": 1707047264859, + "lastObserved": 1707327443020, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -410,11 +414,11 @@ "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" + "container": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" } }, "systemMetadata": { - "lastObserved": 1707047264860, + "lastObserved": 1707327443022, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -439,7 +443,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264861, + "lastObserved": 1707327443023, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -457,14 +461,14 @@ "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" }, { - "id": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", - "urn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" } ] } }, "systemMetadata": { - "lastObserved": 1707047264861, + "lastObserved": 1707327443024, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -480,7 +484,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264862, + "lastObserved": 1707327443025, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -492,9 +496,12 @@ "aspectName": "chartInfo", "aspect": { "json": { - "customProperties": {}, - "title": "QYUUb", - "description": "barchart", + "customProperties": { + "Dimension": "[AxisProperty(Title='City', Min='NaN', Max='NaN')]", + "Measure": "[AxisProperty(Title='Sum(Date)', Min='89411', Max='2144260')]" + }, + "title": "Test_chart", + "description": "scatterplot", "lastModified": { "created": { "time": 0, @@ -508,14 +515,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264863, + "lastObserved": 1707327443026, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -524,19 +531,19 @@ } }, "systemMetadata": { - "lastObserved": 1707047264865, + "lastObserved": 1707327443028, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "harshal-playground-306419.test_dataset.test_table", + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje", "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { @@ -570,19 +577,20 @@ } }, "systemMetadata": { - "lastObserved": 1707047264866, + "lastObserved": 1707327443029, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { "json": { "customProperties": { + "type": "Qlik Table", "dataconnectorid": "bb5be407-d3d3-4f19-858c-e71d593f09ae", "dataconnectorName": "Google_BigQuery_harshal-playground-306419" }, @@ -592,30 +600,30 @@ } }, "systemMetadata": { - "lastObserved": 1707047264867, + "lastObserved": 1707327443031, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" + "container": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" } }, "systemMetadata": { - "lastObserved": 1707047264868, + "lastObserved": 1707327443032, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -626,14 +634,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264869, + "lastObserved": 1707327443033, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -651,14 +659,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264869, + "lastObserved": 1707327443034, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -669,21 +677,21 @@ "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" }, { - "id": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", - "urn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" } ] } }, "systemMetadata": { - "lastObserved": 1707047264870, + "lastObserved": 1707327443035, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -692,19 +700,200 @@ } }, "systemMetadata": { - "lastObserved": 1707047264872, + "lastObserved": 1707327443037, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "659d8aef92df266ef3aa5a7c", + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443038, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "type": "Qlik Table", + "dataconnectorid": "87d7bc7e-77d8-40dc-a251-3a35ec107b4e", + "dataconnectorName": "DataFiles" + }, + "name": "IPL_Matches_2022", + "qualifiedName": "IPL_Matches_2022", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443040, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + }, + "systemMetadata": { + "lastObserved": 1707327443041, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443042, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.IPL_Matches_2022.csv,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443043, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443045, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443048, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv", "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { @@ -774,14 +963,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264872, + "lastObserved": 1707327443050, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -789,10 +978,12 @@ "customProperties": { "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", "spaceId": "659d0e41d1b0ecce6eebc9b1", - "type": "DELIMETED", + "type": "Qlik Dataset", + "datasetType": "DELIMETED", "size": "38168", "rowCount": "74" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/659d8aef12794f37026cb262", "name": "IPL_Matches_2022.csv", "qualifiedName": "IPL_Matches_2022.csv", "description": "", @@ -806,14 +997,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264874, + "lastObserved": 1707327443055, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -822,14 +1013,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264875, + "lastObserved": 1707327443057, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -847,14 +1038,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264876, + "lastObserved": 1707327443058, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -865,14 +1056,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264877, + "lastObserved": 1707327443060, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -886,14 +1077,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264878, + "lastObserved": 1707327443062, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -902,19 +1093,19 @@ } }, "systemMetadata": { - "lastObserved": 1707047264880, + "lastObserved": 1707327443065, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "65a137c849f82a37c625151b", + "schemaName": "personal-space-id.test_tabl", "platform": "urn:li:dataPlatform:qlik-sense", "version": 0, "created": { @@ -948,14 +1139,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264881, + "lastObserved": 1707327443066, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -963,10 +1154,12 @@ "customProperties": { "qri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", "spaceId": "personal-space-id", - "type": "CONNECTION_BASED_DATASET", + "type": "Qlik Dataset", + "datasetType": "CONNECTION_BASED_DATASET", "size": "0", "rowCount": "1" }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/65a137c8d5a03b02d359624a", "name": "test_tabl", "qualifiedName": "test_tabl", "description": "", @@ -980,14 +1173,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264882, + "lastObserved": 1707327443069, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -996,14 +1189,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264883, + "lastObserved": 1707327443071, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -1021,14 +1214,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264884, + "lastObserved": 1707327443073, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1039,14 +1232,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264885, + "lastObserved": 1707327443073, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1060,7 +1253,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264886, + "lastObserved": 1707327443074, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1076,7 +1269,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264886, + "lastObserved": 1707327443076, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1092,14 +1285,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264887, + "lastObserved": 1707327443076, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1113,7 +1306,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264888, + "lastObserved": 1707327443078, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1131,21 +1324,46 @@ "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" }, { - "id": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", - "urn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443079, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" } ] } }, "systemMetadata": { - "lastObserved": 1707047264888, + "lastObserved": 1707327443080, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,harshal-playground-306419.test_dataset.test_table,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1156,21 +1374,21 @@ "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" }, { - "id": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d", - "urn": "urn:li:container:032d02d88d0c25e289e88c688ba84d1d" + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" } ] } }, "systemMetadata": { - "lastObserved": 1707047264889, + "lastObserved": 1707327443081, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d8aef92df266ef3aa5a7c,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1184,14 +1402,14 @@ } }, "systemMetadata": { - "lastObserved": 1707047264890, + "lastObserved": 1707327443082, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,65a137c849f82a37c625151b,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1205,7 +1423,7 @@ } }, "systemMetadata": { - "lastObserved": 1707047264891, + "lastObserved": 1707327443082, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py index a0bbdc4348e1b..c5a14c8d7432a 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py +++ b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, Dict from unittest.mock import patch import pytest @@ -8,7 +8,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: - api_vs_response = { + api_vs_response: Dict[str, Dict] = { "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/api-keys": { "method": "GET", "status_code": 200, @@ -411,17 +411,156 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: }, }, }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/actions/expand?node=qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647&level=TABLE": { + "method": "GET", + "status_code": 200, + "json": { + "graph": { + "id": "", + "directed": True, + "type": "TABLE", + "label": "Expansion for qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647", + "nodes": { + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc": { + "label": "IPL_Matches_2022", + "metadata": {"fields": 1, "type": "TABLE"}, + }, + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE": { + "label": "test_table", + "metadata": {"fields": 1, "type": "TABLE"}, + }, + }, + "edges": [ + { + "relation": "from", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#t91dFUED_ebvKHOjauxSOOCnlRZQTwEkNHv_bjUl7AY#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + }, + ], + "metadata": {}, + } + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/actions/expand?node=qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647%23FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc&level=FIELD": { + "method": "GET", + "status_code": 200, + "json": { + "graph": { + "id": "", + "directed": True, + "type": "FIELD", + "label": "Expansion for qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + "nodes": { + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#-NAdo6895jVLliOXp13Do7e1kDNDDPwxZ0CDuGFJb8A": { + "label": "City", + "metadata": {"type": "FIELD"}, + }, + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#3DAdoo7e1kDNDDPwxZ0CD3Do7e1kDNDDPwxZ0CDuGFJ": { + "label": "Date", + "metadata": {"type": "FIELD"}, + }, + }, + "edges": [ + { + "relation": "from", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#fpA_oxHhnvRu-RrTC_INPo-TEykis1Y_6e-Qy0rKy2I#ti1Pyd82WD92n5Mafl23Gs1Pr_pwJ1tyFIWtaTJfslc#BcCb_mytkkVA-HSHbFLRYOHFY-O_55m8X_JG7DNzMNE", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#-NAdo6895jVLliOXp13Do7e1kDNDDPwxZ0CDuGFJb8A", + }, + { + "relation": "from", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#fpA_oxHhnvRu-RrTC_INPo-TEykis1Y_6e-Qy0rKy2I#ti1Pyd82WD92n5Mafl23Gs1Pr_pwJ1tyFIWtaTJfslc#BcCb_mytkkVA-HSHbFLRYOHFY-O_58X8X_JG7DNzMNE", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#3DAdoo7e1kDNDDPwxZ0CD3Do7e1kDNDDPwxZ0CDuGFJ", + }, + ], + "metadata": {}, + } + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/actions/expand?node=qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647%23Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE&level=FIELD": { + "method": "GET", + "status_code": 200, + "json": { + "graph": { + "id": "", + "directed": True, + "type": "FIELD", + "label": "Expansion for qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE", + "nodes": { + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok": { + "label": "name", + "metadata": {"type": "FIELD"}, + } + }, + "edges": [ + { + "relation": "read", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#JOKG8u7CvizvGXwrFsyXRU0yKr2rL2WFD5djpH9bj5Q#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok", + } + ], + "metadata": {}, + } + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/overview": { + "method": "POST", + "response_list": [ + { + "status_code": 200, + "json": { + "resources": [ + { + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#-NAdo6895jVLliOXp13Do7e1kDNDDPwxZ0CDuGFJb8A", + "lineage": [ + { + "resourceLabel": "d7a6c03238b0c25b3d5c8631e1121807.qvd", + "resourceQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#_s2Ws5RVxAArzAZlmghwjHNGq8ZnjagxptAl6jlZOaw", + "tableQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#_s2Ws5RVxAArzAZlmghwjHNGq8ZnjagxptAl6jlZOaw#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + "tableLabel": "IPL_Matches_2022", + }, + ], + } + ] + }, + }, + { + "status_code": 200, + "json": { + "resources": [ + { + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok", + "lineage": [ + { + "resourceLabel": "3fcbce17de9e55774ddedc345532c419.qvd", + "resourceQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#JOKG8u7CvizvGXwrFsyXRU0yKr2rL2WFD5djpH9bj5Q", + "tableQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#JOKG8u7CvizvGXwrFsyXRU0yKr2rL2WFD5djpH9bj5Q#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE", + "tableLabel": "test_table", + } + ], + } + ] + }, + }, + ], + }, } api_vs_response.update(override_data) for url in api_vs_response.keys(): - request_mock.register_uri( - api_vs_response[url]["method"], - url, - json=api_vs_response[url]["json"], - status_code=api_vs_response[url]["status_code"], - ) + if api_vs_response[url].get("response_list"): + request_mock.register_uri( + api_vs_response[url]["method"], + url, + response_list=api_vs_response[url]["response_list"], + ) + else: + request_mock.register_uri( + api_vs_response[url]["method"], + url, + json=api_vs_response[url]["json"], + status_code=api_vs_response[url]["status_code"], + ) def mock_websocket_response(*args, **kwargs): @@ -450,7 +589,7 @@ def mock_websocket_response(*args, **kwargs): return { "qLayout": { "qTitle": "IPL_Matches_2022", - "qFileName": "e09a68e7-18c9-461d-b957-043e0c045dcd", + "qFileName": "f0714ca7-7093-49e4-8b58-47bb38563647", "qLastReloadTime": "2024-01-15T11:06:53.070Z", "qHasScript": True, "qStateNames": [], @@ -460,7 +599,7 @@ def mock_websocket_response(*args, **kwargs): "qUnsupportedFeatures": [], "qUsage": "ANALYTICS", "encrypted": True, - "id": "e09a68e7-18c9-461d-b957-043e0c045dcd", + "id": "f0714ca7-7093-49e4-8b58-47bb38563647", "published": False, "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", "ownerId": "657b5abe656297cec3d8b205", @@ -595,6 +734,108 @@ def mock_websocket_response(*args, **kwargs): elif request == { "jsonrpc": "2.0", "id": 6, + "handle": 2, + "method": "GetChild", + "params": {"qId": "QYUUb"}, + }: + return { + "qReturn": { + "qType": "GenericObject", + "qHandle": 3, + "qGenericType": "scatterplot", + "qGenericId": "QYUUb", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 7, + "handle": 3, + "method": "GetLayout", + "params": {}, + }: + return { + "qLayout": { + "qInfo": {"qId": "QYUUb", "qType": "scatterplot"}, + "qMeta": {"privileges": ["read", "update", "delete"]}, + "qSelectionInfo": {}, + "qHyperCube": { + "qSize": {"qcx": 3, "qcy": 5}, + "qDimensionInfo": [ + { + "qFallbackTitle": "City", + "qApprMaxGlyphCount": 11, + "qCardinal": 5, + "qSortIndicator": "A", + "qGroupFallbackTitles": ["City"], + "qGroupPos": 0, + "qStateCounts": { + "qLocked": 0, + "qSelected": 0, + "qOption": 5, + "qDeselected": 0, + "qAlternative": 0, + "qExcluded": 0, + "qSelectedExcluded": 0, + "qLockedExcluded": 0, + }, + "qTags": [ + "$ascii", + "$text", + "$geoname", + "$relates_IPL_Matches_2022.City_GeoInfo", + ], + "qDimensionType": "D", + "qGrouping": "N", + "qNumFormat": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + "qIsAutoFormat": True, + "qGroupFieldDefs": ["City"], + "qMin": "NaN", + "qMax": "NaN", + "qAttrExprInfo": [], + "qAttrDimInfo": [], + "qCardinalities": { + "qCardinal": 5, + "qHypercubeCardinal": 5, + "qAllValuesCardinal": -1, + }, + "autoSort": True, + "cId": "FdErA", + "othersLabel": "Others", + } + ], + "qMeasureInfo": [ + { + "qFallbackTitle": "Sum(Date)", + "qApprMaxGlyphCount": 7, + "qCardinal": 0, + "qSortIndicator": "D", + "qNumFormat": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + "qMin": 89411, + "qMax": 2144260, + "qIsAutoFormat": True, + "qAttrExprInfo": [], + "qAttrDimInfo": [], + "qTrendLines": [], + "autoSort": True, + "cId": "PtTpyG", + "numFormatFromTemplate": True, + }, + ], + }, + "script": "", + "showTitles": True, + "title": "Test_chart", + "subtitle": "", + "footnote": "", + "disableNavMenu": False, + "showDetails": True, + "showDetailsExpression": False, + "visualization": "scatterplot", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 8, "handle": 1, "method": "GetObject", "params": ["LoadModel"], @@ -602,15 +843,15 @@ def mock_websocket_response(*args, **kwargs): return { "qReturn": { "qType": "GenericObject", - "qHandle": 3, + "qHandle": 4, "qGenericType": "LoadModel", "qGenericId": "LoadModel", } } elif request == { "jsonrpc": "2.0", - "id": 7, - "handle": 3, + "id": 9, + "handle": 4, "method": "GetLayout", "params": {}, }: @@ -689,15 +930,51 @@ def mock_websocket_response(*args, **kwargs): }, "selectStatement": "SELECT name\nFROM `harshal-playground-306419`.`test_dataset`.`test_table`;", "caching": {"enabled": True, "type": "qvd"}, - } + }, + { + "dataconnectorName": "DataFiles", + "dataconnectorPrefix": "test_space:", + "boxType": "load-file", + "databaseName": "IPL_Matches_2022.csv", + "ownerName": "TYPE9_CSV", + "tableName": "IPL_Matches_2022", + "tableAlias": "IPL_Matches_2022", + "key": "DataFiles:IPL_Matches_2022.csv:TYPE9_CSV:IPL_Matches_2022", + "fields": [ + { + "id": "dsd.IPL_Matches_2022.City", + "name": "City", + "alias": "City", + "selected": True, + }, + { + "id": "dsd.IPL_Matches_2022.Date", + "name": "Date", + "alias": "Date", + "selected": True, + }, + ], + "connectionInfo": { + "type": {"isStorageProvider": False}, + "id": "87d7bc7e-77d8-40dc-a251-3a35ec107b4e", + "name": "DataFiles", + "typeName": "qix-datafiles.exe", + "sourceConnectorID": "qix-datafiles.exe", + "connectionString": 'CUSTOM CONNECT TO "provider=qix-datafiles.exe;path=test_space:datafiles;"', + "space": "659d0e41d1b0ecce6eebc9b1", + "dataconnectorPrefix": "test_space:", + "caching": {"enabled": True, "type": "qvd"}, + }, + "id": "dsd.IPL_Matches_2022", + "loadSelectStatement": "LOAD [ID] AS [ID],\n\t[City] AS [City],\n\t[Date] AS [Date],\n\t[Season] AS [Season],\n\t[MatchNumber] AS [MatchNumber],\n\t[Team1] AS [Team1],\n\t[Team2] AS [Team2],\n\t[Venue] AS [Venue],\n\t[TossWinner] AS [TossWinner],\n\t[TossDecision] AS [TossDecision],\n\t[SuperOver] AS [SuperOver],\n\t[WinningTeam] AS [WinningTeam],\n\t[WonBy] AS [WonBy],\n\t[Margin] AS [Margin],\n\t[method] AS [method],\n\t[Player_of_Match] AS [Player_of_Match],\n\t[Team1Players] AS [Team1Players],\n\t[Team2Players] AS [Team2Players],\n\t[Umpire1] AS [Umpire1],\n\t[Umpire2] AS [Umpire2]\nFROM [lib://DataFiles/IPL_Matches_2022.csv]\n(txt, codepage is 28591, embedded labels, delimiter is ',', msq);\n", + "formatSpec": "(txt, codepage is 28591, embedded labels, delimiter is ',', msq)", + "caching": {"enabled": True, "type": "qvd"}, + }, ], "schemaVersion": 2.1, } } else: - import pdb - - pdb.set_trace() return {} From 191f0ff9e8cfa60a7049ac134d0f65e055d486f2 Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Thu, 8 Feb 2024 17:22:28 +0530 Subject: [PATCH 09/15] Add fineGrained Lineage for app used tables and datasets --- .../ingestion/source/qlik_sense/qlik_api.py | 8 +- .../ingestion/source/qlik_sense/qlik_sense.py | 37 ++++++-- .../golden_test_platform_instance_ingest.json | 88 ++++++++++++++++++- .../golden_test_qlik_sense_ingest.json | 84 +++++++++++++++++- 4 files changed, 202 insertions(+), 15 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py index d5d559144ef16..76a7d4ce60af3 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py @@ -125,8 +125,9 @@ def _get_sheet( response = websocket_connection.websocket_send_request(method="GetLayout") sheet_dict = response[Constant.QLAYOUT] sheet = Sheet.parse_obj(sheet_dict[Constant.QMETA]) - i = 1 - for chart_dict in sheet_dict[Constant.QCHILDLIST][Constant.QITEMS]: + for i, chart_dict in enumerate( + sheet_dict[Constant.QCHILDLIST][Constant.QITEMS] + ): chart = self._get_chart( websocket_connection, chart_dict[Constant.QINFO][Constant.QID], @@ -134,8 +135,7 @@ def _get_sheet( ) if chart: if not chart.title: - chart.title = f"Object {i}" - i += 1 + chart.title = f"Object {i+1} of Sheet '{sheet.title}'" sheet.charts.append(chart) websocket_connection.handle.pop() return sheet diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py index 3cdc5c846dd79..83f564a13f819 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py @@ -61,6 +61,9 @@ from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( DatasetLineageType, DatasetProperties, + FineGrainedLineage, + FineGrainedLineageDownstreamType, + FineGrainedLineageUpstreamType, Upstream, UpstreamLineage, ) @@ -244,7 +247,9 @@ def _gen_dashboard_info_workunit( entityUrn=dashboard_urn, aspect=dashboard_info_cls ).as_workunit() - def _gen_charts_workunit(self, charts: List[Chart]) -> Iterable[MetadataWorkUnit]: + def _gen_charts_workunit( + self, charts: List[Chart], app_id: str + ) -> Iterable[MetadataWorkUnit]: """ Map Qlik Chart to Datahub Chart """ @@ -272,6 +277,12 @@ def _gen_charts_workunit(self, charts: List[Chart]) -> Iterable[MetadataWorkUnit ), ).as_workunit() + yield from add_entity_to_container( + container_key=self._gen_app_key(app_id), + entity_type="chart", + entity_urn=chart_urn, + ) + def _gen_sheets_workunit( self, sheets: List[Sheet], app_id: str ) -> Iterable[MetadataWorkUnit]: @@ -299,7 +310,7 @@ def _gen_sheets_workunit( if self.config.ingest_owner and owner_username: yield self._gen_entity_owner_aspect(dashboard_urn, owner_username) - yield from self._gen_charts_workunit(sheet.charts) + yield from self._gen_charts_workunit(sheet.charts, app_id) def _gen_app_table_upstream_lineage( self, dataset_urn: str, table: QlikTable @@ -316,7 +327,7 @@ def _gen_app_table_upstream_lineage( ) ) upstream_dataset_urn = builder.make_dataset_urn_with_platform_instance( - name=f"{table.databaseName}.{table.schemaName}.{table.tableName}", + name=f"{table.databaseName}.{table.schemaName}.{table.tableName}".lower(), platform=KNOWN_DATA_PLATFORM_MAPPING.get( table.dataconnectorPlatform, table.dataconnectorPlatform ), @@ -325,9 +336,24 @@ def _gen_app_table_upstream_lineage( ) elif table.type == BoxType.LOADFILE: upstream_dataset_urn = self._gen_qlik_dataset_urn( - f"{table.spaceId}.{table.databaseName}" + f"{table.spaceId}.{table.databaseName}".lower() ) + if upstream_dataset_urn: + # Generate finegrained lineage + fine_grained_lineages = [ + FineGrainedLineage( + upstreamType=FineGrainedLineageUpstreamType.FIELD_SET, + upstreams=[ + builder.make_schema_field_urn(upstream_dataset_urn, field.name) + ], + downstreamType=FineGrainedLineageDownstreamType.FIELD, + downstreams=[ + builder.make_schema_field_urn(dataset_urn, field.name) + ], + ) + for field in table.datasetSchema + ] return MetadataChangeProposalWrapper( entityUrn=dataset_urn, aspect=UpstreamLineage( @@ -335,7 +361,8 @@ def _gen_app_table_upstream_lineage( Upstream( dataset=upstream_dataset_urn, type=DatasetLineageType.COPY ) - ] + ], + fineGrainedLineages=fine_grained_lineages, ), ).as_workunit() else: diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json index 7cfe225dddbd7..7558d08b9e3f5 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json @@ -561,6 +561,51 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,qlik_sense_platform.QYUUb)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + }, + "systemMetadata": { + "lastObserved": 1707393018679, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,qlik_sense_platform.QYUUb)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707393018680, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", @@ -713,11 +758,24 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,google-cloud.harshal-playground-306419.test_dataset.test_table,DEV)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,google-cloud.harshal-playground-306419.test_dataset.test_table,DEV),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD),name)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { - "lastObserved": 1707327546144, + "lastObserved": 1707393018687, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -912,14 +970,38 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.IPL_Matches_2022.csv,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD),City)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD),City)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD),Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD),Date)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { - "lastObserved": 1707327546152, + "lastObserved": 1707393018696, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json index 32af5d2191de4..18542a1e4c4ef 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json @@ -520,6 +520,47 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,QYUUb)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + }, + "systemMetadata": { + "lastObserved": 1707393018435, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,QYUUb)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707393018436, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", @@ -655,11 +696,24 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_dataset.test_table,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_dataset.test_table,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD),name)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { - "lastObserved": 1707327443034, + "lastObserved": 1707393018442, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -833,14 +887,38 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.IPL_Matches_2022.csv,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD),City)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD),City)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD),Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD),Date)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { - "lastObserved": 1707327443043, + "lastObserved": 1707393018451, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } From 6f97cc5b366ffcc6fb3c91b60f9f3df52f304c36 Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Tue, 13 Feb 2024 11:22:22 +0530 Subject: [PATCH 10/15] Add app used tables as upstreams of all charts --- .../ingestion/source/qlik_sense/qlik_sense.py | 23 +++++++++++-------- .../golden_test_platform_instance_ingest.json | 12 ++++++++-- .../golden_test_qlik_sense_ingest.json | 12 ++++++++-- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py index 83f564a13f819..a729bc1bd7c9a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py @@ -248,11 +248,17 @@ def _gen_dashboard_info_workunit( ).as_workunit() def _gen_charts_workunit( - self, charts: List[Chart], app_id: str + self, charts: List[Chart], input_tables: List[QlikTable], app_id: str ) -> Iterable[MetadataWorkUnit]: """ Map Qlik Chart to Datahub Chart """ + input_tables_urns: List[str] = [] + for table in input_tables: + table_identifier = self._get_app_table_identifier(table) + if table_identifier: + input_tables_urns.append(self._gen_qlik_dataset_urn(table_identifier)) + for chart in charts: chart_urn = builder.make_chart_urn( platform=self.platform, @@ -274,6 +280,7 @@ def _gen_charts_workunit( description=chart.visualization, lastModified=ChangeAuditStampsClass(), customProperties=custom_properties, + inputs=input_tables_urns, ), ).as_workunit() @@ -283,21 +290,19 @@ def _gen_charts_workunit( entity_urn=chart_urn, ) - def _gen_sheets_workunit( - self, sheets: List[Sheet], app_id: str - ) -> Iterable[MetadataWorkUnit]: + def _gen_sheets_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: """ Map Qlik Sheet to Datahub dashboard """ - for sheet in sheets: + for sheet in app.sheets: dashboard_urn = self._gen_dashboard_urn(sheet.id) yield self._gen_entity_status_aspect(dashboard_urn) - yield self._gen_dashboard_info_workunit(sheet, app_id) + yield self._gen_dashboard_info_workunit(sheet, app.id) yield from add_entity_to_container( - container_key=self._gen_app_key(app_id), + container_key=self._gen_app_key(app.id), entity_type="dashboard", entity_urn=dashboard_urn, ) @@ -310,7 +315,7 @@ def _gen_sheets_workunit( if self.config.ingest_owner and owner_username: yield self._gen_entity_owner_aspect(dashboard_urn, owner_username) - yield from self._gen_charts_workunit(sheet.charts, app_id) + yield from self._gen_charts_workunit(sheet.charts, app.tables, app.id) def _gen_app_table_upstream_lineage( self, dataset_urn: str, table: QlikTable @@ -446,7 +451,7 @@ def _gen_app_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: created=int(app.createdAt.timestamp() * 1000), last_modified=int(app.updatedAt.timestamp() * 1000), ) - yield from self._gen_sheets_workunit(app.sheets, app.id) + yield from self._gen_sheets_workunit(app) yield from self._gen_app_tables_workunit(app.tables, app.id) def _gen_qlik_dataset_urn(self, dataset_identifier: str) -> str: diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json index 7558d08b9e3f5..6917b833d05a3 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json @@ -552,11 +552,19 @@ "time": 0, "actor": "urn:li:corpuser:unknown" } - } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)" + } + ] } }, "systemMetadata": { - "lastObserved": 1707327546138, + "lastObserved": 1707803447230, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json index 18542a1e4c4ef..5b47c09a79143 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json @@ -511,11 +511,19 @@ "time": 0, "actor": "urn:li:corpuser:unknown" } - } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)" + } + ] } }, "systemMetadata": { - "lastObserved": 1707327443026, + "lastObserved": 1707803447013, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } From 90b87909262df826d191ea4f4f6cedf20dd753bd Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Wed, 21 Feb 2024 00:13:14 +0530 Subject: [PATCH 11/15] Add Websocket-client package in qlik-sense plugin --- metadata-ingestion/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 3284254a206c0..43b66a370eddf 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -395,7 +395,7 @@ # databricks is alias for unity-catalog and needs to be kept in sync "databricks": databricks | sql_common | sqllineage_lib, "fivetran": snowflake_common | bigquery_common, - "qlik-sense": {"requests"}, + "qlik-sense": {"requests", "websocket-client"}, } # This is mainly used to exclude plugins from the Docker image. @@ -525,6 +525,7 @@ "mode", "fivetran", "kafka-connect", + "qlik-sense", ] if plugin for dependency in plugins[plugin] From 2c4931a2f55d1fa373f68efa952bbfbf63b3dd6f Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Wed, 21 Feb 2024 16:32:39 +0530 Subject: [PATCH 12/15] Big fixed to fetch all items present --- .../ingestion/source/qlik_sense/config.py | 3 ++ .../ingestion/source/qlik_sense/qlik_api.py | 50 +++++++++++-------- .../integration/qlik_sense/test_qlik_sense.py | 5 ++ 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py index 1bcd89dfcb491..de4e59e34a1a9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py @@ -32,6 +32,9 @@ class Constant: NAME = "name" TYPE = "type" ITEMID = "itemId" + NEXT = "next" + LINKS = "links" + HREF = "href" DATASETTYPE = "datasetType" CREATEDAT = "createdAt" UPDATEDAT = "updatedAt" diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py index 76a7d4ce60af3..f1bd9a028383a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py @@ -256,28 +256,34 @@ def _get_app(self, app_id: str) -> Optional[App]: def get_items(self) -> List[Item]: items: List[Item] = [] try: - response = self.session.get(f"{self.rest_api_url}/items") - response.raise_for_status() - data = response.json()[Constant.DATA] - for item in data: - # spaceId none indicates item present in personal space - if not item.get(Constant.SPACEID): - item[Constant.SPACEID] = Constant.PERSONAL_SPACE_ID - if self.config.space_pattern.allowed( - self.spaces[item[Constant.SPACEID]] - ): - resource_type = item[Constant.RESOURCETYPE] - if resource_type == Constant.APP: - app = self._get_app(app_id=item[Constant.RESOURCEID]) - if app: - items.append(app) - elif resource_type == Constant.DATASET: - dataset = self._get_dataset( - dataset_id=item[Constant.RESOURCEID], - item_id=item[Constant.ID], - ) - if dataset: - items.append(dataset) + url = f"{self.rest_api_url}/items" + while True: + response = self.session.get(url) + response.raise_for_status() + response_dict = response.json() + for item in response_dict[Constant.DATA]: + # spaceId none indicates item present in personal space + if not item.get(Constant.SPACEID): + item[Constant.SPACEID] = Constant.PERSONAL_SPACE_ID + if self.config.space_pattern.allowed( + self.spaces[item[Constant.SPACEID]] + ): + resource_type = item[Constant.RESOURCETYPE] + if resource_type == Constant.APP: + app = self._get_app(app_id=item[Constant.RESOURCEID]) + if app: + items.append(app) + elif resource_type == Constant.DATASET: + dataset = self._get_dataset( + dataset_id=item[Constant.RESOURCEID], + item_id=item[Constant.ID], + ) + if dataset: + items.append(dataset) + if Constant.NEXT in response_dict[Constant.LINKS]: + url = response_dict[Constant.LINKS][Constant.NEXT][Constant.HREF] + else: + break except Exception as e: self._log_http_error(message=f"Unable to fetch items. Exception: {e}") diff --git a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py index c5a14c8d7432a..00a50b7dbe54c 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py +++ b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py @@ -184,6 +184,11 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "resourceSize": {"appFile": 0, "appMemory": 0}, }, ], + "links": { + "self": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items" + } + }, }, }, "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/users/657b5abe656297cec3d8b205": { From ae7bd0b3f160cccd8dcb5a3f25ae47262d051d0c Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Thu, 22 Feb 2024 18:40:38 +0530 Subject: [PATCH 13/15] Fix bug for lineage metadata --- metadata-ingestion/setup.py | 2 +- .../source/qlik_sense/data_classes.py | 12 ++----- .../ingestion/source/qlik_sense/qlik_api.py | 34 ++++++++++++------- .../ingestion/source/qlik_sense/qlik_sense.py | 10 ++++-- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 43b66a370eddf..23ae01d38cdd9 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -395,7 +395,7 @@ # databricks is alias for unity-catalog and needs to be kept in sync "databricks": databricks | sql_common | sqllineage_lib, "fivetran": snowflake_common | bigquery_common, - "qlik-sense": {"requests", "websocket-client"}, + "qlik-sense": {"requests", "websocket-client"} | sqlglot_lib, } # This is mainly used to exclude plugins from the Docker image. diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py index a7c6f76464b63..c30b456253e06 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py @@ -191,7 +191,6 @@ def update_values(cls, values: Dict) -> Dict: class QlikTable(BaseModel): tableName: str - databaseName: str type: BoxType = Field(alias="boxType") tableAlias: str dataconnectorid: str @@ -200,22 +199,17 @@ class QlikTable(BaseModel): spaceId: str datasetSchema: List[SchemaField] = Field(alias="fields") tableQri: Optional[str] = None + selectStatement: Optional[str] = None + databaseName: Optional[str] = None schemaName: Optional[str] = None @root_validator(pre=True) def update_values(cls, values: Dict) -> Dict: - if values["boxType"] == BoxType.BLACKBOX.value: - values[Constant.DATABASENAME] = values[Constant.CONNECTORPROPERTIES][ - Constant.TABLEQUALIFIERS - ][0] - values[Constant.SCHEMANAME] = values[Constant.CONNECTORPROPERTIES][ - Constant.TABLEQUALIFIERS - ][1] values[Constant.DATACONNECTORID] = values[Constant.CONNECTIONINFO][Constant.ID] values[Constant.DATACONNECTORPLATFORM] = values[Constant.CONNECTIONINFO][ Constant.SOURCECONNECTORID ] - values[Constant.SPACEID] = values[Constant.CONNECTIONINFO]["space"] + values[Constant.SPACEID] = values[Constant.CONNECTIONINFO][Constant.SPACE] return values diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py index f1bd9a028383a..abcd0c0ce8353 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py @@ -124,6 +124,9 @@ def _get_sheet( ) response = websocket_connection.websocket_send_request(method="GetLayout") sheet_dict = response[Constant.QLAYOUT] + if Constant.OWNERID not in sheet_dict[Constant.QMETA]: + # That means sheet is private sheet + return None sheet = Sheet.parse_obj(sheet_dict[Constant.QMETA]) for i, chart_dict in enumerate( sheet_dict[Constant.QCHILDLIST][Constant.QITEMS] @@ -159,20 +162,27 @@ def _add_qri_of_tables(self, tables: List[QlikTable], app_id: str) -> None: f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/actions/expand?node={table_node_qri}&level=FIELD" ) response.raise_for_status() - field_node_qri = list( + field_nodes_qris = list( response.json()[Constant.GRAPH][Constant.NODES].keys() - )[0] - response = self.session.post( - f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/overview", - json=[field_node_qri], ) - response.raise_for_status() - for each_lineage in response.json()[Constant.RESOURCES][0][ - Constant.LINEAGE - ]: - table_qri_dict[each_lineage[Constant.TABLELABEL]] = each_lineage[ - Constant.TABLEQRI - ] + for field_node_qri in field_nodes_qris: + response = self.session.post( + f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/overview", + json=[field_node_qri], + ) + response.raise_for_status() + # Some fields might not have lineage overview, in that case status code is 207 + if response.status_code == 200: + for each_lineage in response.json()[Constant.RESOURCES][0][ + Constant.LINEAGE + ]: + table_name = ( + each_lineage[Constant.TABLELABEL] + .replace('"', "") + .split(".")[-1] + ) + table_qri_dict[table_name] = each_lineage[Constant.TABLEQRI] + break for table in tables: if table.tableName in table_qri_dict: table.tableQri = table_qri_dict[table.tableName] diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py index a729bc1bd7c9a..a5b9adae0376c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py @@ -84,6 +84,7 @@ OwnershipTypeClass, SchemaFieldDataTypeClass, ) +from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result # Logger instance logger = logging.getLogger(__name__) @@ -331,14 +332,17 @@ def _gen_app_table_upstream_lineage( ), ) ) - upstream_dataset_urn = builder.make_dataset_urn_with_platform_instance( - name=f"{table.databaseName}.{table.schemaName}.{table.tableName}".lower(), + if not table.selectStatement: + return None + upstream_dataset_urn = create_lineage_sql_parsed_result( + query=table.selectStatement.strip(), + default_db=None, platform=KNOWN_DATA_PLATFORM_MAPPING.get( table.dataconnectorPlatform, table.dataconnectorPlatform ), env=upstream_dataset_platform_detail.env, platform_instance=upstream_dataset_platform_detail.platform_instance, - ) + ).in_tables[0] elif table.type == BoxType.LOADFILE: upstream_dataset_urn = self._gen_qlik_dataset_urn( f"{table.spaceId}.{table.databaseName}".lower() From 224b6566e5691571ece621f6d789e7557ba2ec9e Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Thu, 22 Feb 2024 18:41:27 +0530 Subject: [PATCH 14/15] Add remained constants --- .../src/datahub/ingestion/source/qlik_sense/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py index de4e59e34a1a9..eb0b9d02d865e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py @@ -41,6 +41,7 @@ class Constant: SECUREQRI = "secureQri" QRI = "qri" SPACEID = "spaceId" + SPACE = "space" CREATEDTIME = "createdTime" LASTMODIFIEDTIME = "lastModifiedTime" OPERATIONAL = "operational" @@ -63,6 +64,7 @@ class Constant: LINEAGE = "lineage" TABLELABEL = "tableLabel" TABLEQRI = "tableQRI" + OWNERID = "ownerId" # Websocket response key constants QID = "qId" RESULT = "result" From ac249d63d001da3abbcc28790629b35df8ed7842 Mon Sep 17 00:00:00 2001 From: shubhamjagtap639 Date: Fri, 23 Feb 2024 13:04:28 +0530 Subject: [PATCH 15/15] Update setup.py --- metadata-ingestion/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index ddce696557340..b8babf6bb836d 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -393,7 +393,7 @@ # databricks is alias for unity-catalog and needs to be kept in sync "databricks": databricks | sql_common | sqllineage_lib, "fivetran": snowflake_common | bigquery_common, - "qlik-sense": {"requests", "websocket-client"} | sqlglot_lib, + "qlik-sense": sqlglot_lib | {"requests", "websocket-client"}, } # This is mainly used to exclude plugins from the Docker image.