From aca48f07c7609e544d88974b6b98ff2c2f0d2712 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sun, 1 Sep 2024 22:15:16 +1200 Subject: [PATCH 01/57] Split extension task in to V1 and V2 --- .github/workflows/extension.yml | 10 +- docs/extension.md | 2 +- extension/.gitignore | 2 +- extension/overrides.local.json | 2 +- extension/package-lock.json | 4 +- extension/package.json | 6 +- .../dependabot/dependabotV1}/icon.png | Bin .../dependabot/dependabotV1}/index.ts | 8 +- .../dependabot/dependabotV1}/task.json | 2 +- .../tasks/dependabot/dependabotV2/icon.png | Bin 0 -> 7388 bytes .../tasks/dependabot/dependabotV2/index.ts | 17 ++ .../tasks/dependabot/dependabotV2/task.json | 262 ++++++++++++++++++ .../utils}/IDependabotConfig.ts | 0 .../utils/convertPlaceholder.ts | 0 .../{task => tasks}/utils/extractHostname.ts | 0 .../utils/extractOrganization.ts | 0 .../utils/extractVirtualDirectory.ts | 0 .../utils/getAzureDevOpsAccessToken.ts | 0 .../utils/getDockerImageTag.ts | 0 .../utils/getGithubAccessToken.ts | 0 .../utils/getSharedVariables.ts | 0 .../{task => tasks}/utils/parseConfigFile.ts | 2 +- .../utils/resolveAzureDevOpsIdentities.ts | 0 .../tests/utils/convertPlaceholder.test.ts | 2 +- extension/tests/utils/extractHostname.test.ts | 2 +- .../tests/utils/extractOrganization.test.ts | 2 +- .../utils/extractVirtualDirectory.test.ts | 2 +- extension/tests/utils/parseConfigFile.test.ts | 4 +- .../resolveAzureDevOpsIdentities.test.ts | 2 +- extension/vss-extension.json | 4 +- 30 files changed, 307 insertions(+), 28 deletions(-) rename extension/{task => tasks/dependabot/dependabotV1}/icon.png (100%) rename extension/{task => tasks/dependabot/dependabotV1}/index.ts (97%) rename extension/{task => tasks/dependabot/dependabotV1}/task.json (99%) create mode 100644 extension/tasks/dependabot/dependabotV2/icon.png create mode 100644 extension/tasks/dependabot/dependabotV2/index.ts create mode 100644 extension/tasks/dependabot/dependabotV2/task.json rename extension/{task => tasks/utils}/IDependabotConfig.ts (100%) rename extension/{task => tasks}/utils/convertPlaceholder.ts (100%) rename extension/{task => tasks}/utils/extractHostname.ts (100%) rename extension/{task => tasks}/utils/extractOrganization.ts (100%) rename extension/{task => tasks}/utils/extractVirtualDirectory.ts (100%) rename extension/{task => tasks}/utils/getAzureDevOpsAccessToken.ts (100%) rename extension/{task => tasks}/utils/getDockerImageTag.ts (100%) rename extension/{task => tasks}/utils/getGithubAccessToken.ts (100%) rename extension/{task => tasks}/utils/getSharedVariables.ts (100%) rename extension/{task => tasks}/utils/parseConfigFile.ts (99%) rename extension/{task => tasks}/utils/resolveAzureDevOpsIdentities.ts (100%) diff --git a/.github/workflows/extension.yml b/.github/workflows/extension.yml index c52029ab..f8bdfd40 100644 --- a/.github/workflows/extension.yml +++ b/.github/workflows/extension.yml @@ -60,7 +60,7 @@ jobs: working-directory: '${{ github.workspace }}/extension' - name: Build - run: npm run build:prod + run: npm run build working-directory: '${{ github.workspace }}/extension' - name: Install tfx-cli @@ -74,11 +74,11 @@ jobs: MAJOR_MINOR_PATCH: ${{ steps.gitversion.outputs.majorMinorPatch }} BUILD_NUMBER: ${{ github.run_number }} - - name: Update values in extension/task/task.json + - name: Update values in extension/tasks/dependabot/dependabotV2/task.json run: | - echo "`jq '.version.Major=${{ steps.gitversion.outputs.major }}' extension/task/task.json`" > extension/task/task.json - echo "`jq '.version.Minor=${{ steps.gitversion.outputs.minor }}' extension/task/task.json`" > extension/task/task.json - echo "`jq '.version.Patch=${{ github.run_number }}' extension/task/task.json`" > extension/task/task.json + echo "`jq '.version.Major=${{ steps.gitversion.outputs.major }}' extension/tasks/dependabot/dependabotV2/task.json`" > extension/tasks/dependabot/dependabotV2/task.json + echo "`jq '.version.Minor=${{ steps.gitversion.outputs.minor }}' extension/tasks/dependabot/dependabotV2/task.json`" > extension/tasks/dependabot/dependabotV2/task.json + echo "`jq '.version.Patch=${{ github.run_number }}' extension/tasks/dependabot/dependabotV2/task.json`" > extension/tasks/dependabot/dependabotV2/task.json - name: Create Extension (dev) run: > diff --git a/docs/extension.md b/docs/extension.md index 50af697e..ef3b5061 100644 --- a/docs/extension.md +++ b/docs/extension.md @@ -34,7 +34,7 @@ npm run build:prod To generate the Azure DevOps `.vsix` extension package for testing, you'll first need to [create a publisher account](https://learn.microsoft.com/en-us/azure/devops/extend/publish/overview?view=azure-devops#create-a-publisher) on the [Visual Studio Marketplace Publishing Portal](https://marketplace.visualstudio.com/manage/createpublisher?managePageRedirect=true). After this, override your publisher ID below and generate the extension with: ```bash -npx tfx-cli extension create --overrides-file overrides.local.json --override "{\"publisher\": \"your-publisher-id-here\"}" --json5 +npm run package -- --overrides-file overrides.local.json --override "{\"publisher\": \"your-publisher-id-here\"}" ``` ## Installing the extension diff --git a/extension/.gitignore b/extension/.gitignore index 57c2ca3a..e949d8ed 100644 --- a/extension/.gitignore +++ b/extension/.gitignore @@ -1,4 +1,4 @@ node_modules .taskkey -task/**/*.js +**/*.js *.vsix \ No newline at end of file diff --git a/extension/overrides.local.json b/extension/overrides.local.json index caa39dcf..06b66533 100644 --- a/extension/overrides.local.json +++ b/extension/overrides.local.json @@ -1,5 +1,5 @@ { "id": "dependabot-local", - "version": "0.1.0.6", + "version": "0.2.0.0", "name": "Dependabot (Local)" } diff --git a/extension/package-lock.json b/extension/package-lock.json index 1be2a53f..cf23f5b9 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "dependabot-azure-devops", - "version": "1.0.0", + "version": "2.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dependabot-azure-devops", - "version": "1.0.0", + "version": "2.0.0", "license": "MIT", "dependencies": { "axios": "1.7.5", diff --git a/extension/package.json b/extension/package.json index f7dda3da..a6094861 100644 --- a/extension/package.json +++ b/extension/package.json @@ -1,11 +1,11 @@ { "name": "dependabot-azure-devops", - "version": "1.0.0", + "version": "2.0.0", "description": "Dependabot Azure DevOps task", "main": "''", "scripts": { - "build": "tsc -p .", - "build:prod": "npm run build && cp -r node_modules task/node_modules", + "build": "cp -r node_modules tasks/dependabot/dependabotV1/node_modules && cp -r node_modules tasks/dependabot/dependabotV2/node_modules && tsc -p .", + "package": "npx tfx-cli extension create --json5", "test": "jest" }, "repository": { diff --git a/extension/task/icon.png b/extension/tasks/dependabot/dependabotV1/icon.png similarity index 100% rename from extension/task/icon.png rename to extension/tasks/dependabot/dependabotV1/icon.png diff --git a/extension/task/index.ts b/extension/tasks/dependabot/dependabotV1/index.ts similarity index 97% rename from extension/task/index.ts rename to extension/tasks/dependabot/dependabotV1/index.ts index b2d5ddfc..e1f7ceb2 100644 --- a/extension/task/index.ts +++ b/extension/tasks/dependabot/dependabotV1/index.ts @@ -1,9 +1,9 @@ import * as tl from 'azure-pipelines-task-lib/task'; import { ToolRunner } from 'azure-pipelines-task-lib/toolrunner'; -import { IDependabotRegistry, IDependabotUpdate } from './IDependabotConfig'; -import getSharedVariables from './utils/getSharedVariables'; -import { parseConfigFile } from './utils/parseConfigFile'; -import { resolveAzureDevOpsIdentities } from './utils/resolveAzureDevOpsIdentities'; +import { IDependabotRegistry, IDependabotUpdate } from '../../utils/IDependabotConfig'; +import getSharedVariables from '../../utils/getSharedVariables'; +import { parseConfigFile } from '../../utils/parseConfigFile'; +import { resolveAzureDevOpsIdentities } from '../../utils/resolveAzureDevOpsIdentities'; async function run() { try { diff --git a/extension/task/task.json b/extension/tasks/dependabot/dependabotV1/task.json similarity index 99% rename from extension/task/task.json rename to extension/tasks/dependabot/dependabotV1/task.json index bdb71f26..364cc258 100644 --- a/extension/task/task.json +++ b/extension/tasks/dependabot/dependabotV1/task.json @@ -14,7 +14,7 @@ "demands": [], "version": { "Major": 1, - "Minor": 6, + "Minor": 33, "Patch": 0 }, "instanceNameFormat": "Dependabot", diff --git a/extension/tasks/dependabot/dependabotV2/icon.png b/extension/tasks/dependabot/dependabotV2/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..ffa0fe7cb7148e126d30c2261f5982556261ee01 GIT binary patch literal 7388 zcmc(ERa6{Zu=U{X!FAB!1Hs)L5?sQ-;1(PP2{J(N2?TcuPJ+8@(7{7+k`P=1!9M~F zce4J+|LH#6hhDu_ozs1~s(0D&}g?;@XwEXJs6BE#=WaRyGn91U+LR>P0qA02=2f?4E(^>Z=rmZ z)U*R)J3}L%x&wo1T21DIw^BdA(BnOB^qGGTf2%bS`x%wTp7t*AS)xXOGTw{95;YY< zVe_-Fr4KBw=x&ZnxdGt5qJEy4nmcphA3a&E->tFHLX43c45MH}?pNNF6d|l^lVtD> z_;@YS4OkURc8#mHsignCVv`q&uez$$ecWxIMZ7m;PYvKqu|6l^>;F|bwc8aV1T{8T zCN0r)u|BVPmHxY>`&4(+4HKa_0BDr-1S2}#WQ8Udk8`!G`PfZfx-K3ELv#gJv0y6S zmBfqAk9=mp$)u%Q!B)B6NQ^zA;DfX}@dL9m(^*QJedA1x0j6^_G4`}vQ4qJnC35(Ma@+&zjUg5d6aZ{$m1 z1Zl!WO_Re??K7ePL=DCQ2HG~@&DvVdhIP-om1=j#JCM5%J|=v+an+OmCv6=^(=iR4 zJdyvI8z+AT7>5R@TLYS5tv`h*3`+17%cAjK-6f3i4kIEk0xKpgXl~ZxI4ooQ9mXB? zkcrS?%6@viM>d&*a z3tR)YD?|BnztvSKB!xpIESn;?nS5QgsEn3{$`_ zKb#oAS)T23H*)yJ#Ge@*M+#UFU>rcwzyHEpJbA4hAkB|7RwRihBk8JJc?6?*_{VwPu@$0p{cQ7HK$t&;_ zMuuSC{+7z6;hJk@T${y4v6Habjwr2|`!s{xR_seQOY-3m)feLw5=v+8&S_wR5HvQL z=0DhizsiY-;LY4k+Ks{=pRZ@|H!D3OV*UUoV1eu)cxUY;IgB?<9{?%a0dZFgpBY>& zp0`a|ad(qeDi4hR!n1D}I>Op84Iv?`P{uw)IfIkFlv}>INtFhMQvf0MF$oC#{^w_Wz=N(|97q5~O8)BG(wa;Ay{XL%pK95Y;|6T`BY5TRF~s zfz@&aSDJmpZ1Dp!brO$qJN=FPELJ$gK-Arzd$zNukZgBc0P8A>is!Gw0Q<4g=MUEF z*DB!>s?W7BT9=V(F7?Oh;6zFwgxulM1PGG&`24BlO^!cKTMI=LO!J+uMf59comA=GT=8&Cf)VN)-p<7m|@C^rkiB?^Ej_evPmK>OtK^tTt*Le>u|;rZ_gK3S2u zSk?H6@{c(0osfc*&oPCAAV|*~L9_R$h2Aldxm4?0WIYk{8)3)7XbCN%70@x~za#m7 zJFd276QS?vI8&l?lcr<_m%A+&WQ&EOp?yup`owvu@C~}^54-*tJxSe=xP$uf6C}oe z@Z>F(RfZk1FqUGec&xo{aIu}XBO~UNMyk=GOn@KcdRZ4jk{7-iX*2RNwtwBJfhm3_ z{ZOf{lAHCsR+L2AMSEQmEzR1SRes}JScxrY2*ftLY_Wgh{T#j&AiDC?O;Efil?oB^_saWlv!w~3QI@F?|E!)*2?vq z_dOQMIT#k6>T1oK9Ao!(e)rc{gn4a_#J5>82_`^@e(&VRK!S{nCW84y{w2t-_PZ=G zkv~G)d)pIE4Gn~1QchBc`fMe*|C)`6-5nX>)7?6W&?xmp9fpD@6osr!GB^qkPCXW5 z67`v2d>ENt`>VU1IbtcQJ4>kSM8h5~cCFHYc~^QJ*mp$Rw{2X2U4Myg@E<=d#BlwW z?I|}Dr-l%~VbN7Tmf_U)Z=j&E~y?S-xNMW2u_!b=_! zCYlmyS^7oQ6;%apDSuQTi`crK0Y2NQbk_as`8{O)CV`6Q0)6!L!CpDhOVEt+f-9hb zdyhBw_=0kpQZj3qw-(`<(JII0l#p&(IDsfcioYX(HU3h(d~|B7{vkVu=*1o$FRz^&yIuTy>fAPqO$t>?-2H0U|kj*u- z`C)F3nx*A3Oh0Pm`Z54HTc>kvQH?cq zTiBJE-2KY|6$yy6Br3$qi)Vp6)S7Y-rEXvFc3mKq_%zyziQ|Jb(^|n*le*8pl+JPc zFK)A=9c$obY(pK={LiJ4^qUx%EIRcUYFfyM6NxL-p#FOt$o(C4x z*#vg!2z;IkmABxrPOHy{m2pCa7ccN2M7=_aZr?S2i>PA4F#;nE<7XH_YZV$;8K#H& z6o8d?{T;6m2oWqoCPz%z3!(SgoNH= zpHsB82g>U2dR05seJlbJi|Tqsw4vM(^gk{n&iN0;)! z6jUL~4$a8(5u_35CIofnHx4`RQQFyO*;wFa19cu}f||4ei5)lOIv-;rVYEY-Bx-`i z(;R(dP1+!S%qh~*a142oVxrS{Iy!VmTw1ertoOo}W;=}d=-2?lC>R=Gm<0kD0O8UA zK!pFNCWhkMGNrfOFXtMqRH584>rcpiza6up6X9a(m=p8?vjTO!g~c(Fxd;-y)VJ)F z|BoRZCIt|rGJ;Yd1xasy$%bGC)2`u~_Rk?;v+cTi%#y#@Wo1&v z&8o1ZFl3|js zi=1S~Y-@Ag={G8Ra}L|WE+|C1AU_(Z3%GWK_$!)zmW|b8zwWd5XE-T0y@W-EubV z38!zIY;pI>lSO@dYq;ec9NbnUJd;`(U(-pr+muhH*-xEn*WVdA;>Y}cgqmfRkou_K zX_xCIQwjBYGVz_2?9grrL5%)x`c(}Hq4C*et@`2EOX!HYcho$arLe^OAX9hS=wtU_^IJOO;K5`-pl;fS3W~e zO8(GI!G{!-TE&)?W*u7Qq#!G=6DY~{@S^oP8|@!a=dV7YDma`xXFEh5KgpH5DO@y+^9~*r$X@;J8?01S*00n* z@^hWi(2Gsxd}?Ws;x|wiIK<7GVJrH2W~KF}tUSqNkz0QWx6q)Y>MN`NPDxB6Tsm;; z873Ul^JE!G|EP21dHp@DIvTOG-j|c~G;ARpY>Ujoh1LbC-LWcZqi_5cctDrp%?c z!V9e^|Bc1pa{Fdi#NHUjY_eV|4rmyNI`D12f;@!M9roiN50tMAT*@sWla5t^*BBV>B_JWmFfI&J7j0-mL4Sh@$0%}vpiKos)|pO){y z0t0=N{R)@%MJxg!tAz535toKSUzsywlOv~FHU5?IUgt%A8Fg6WYkSm4nkFbw+(czrmZTpy#O_(P1sj23_qE(qs9dNqR7p-_13L(&QPK!ot*6MHu%_+v90KqLD9XY%cF*r8;hMG+%R<(lCo zvH{BLYd^5jf9Q!S7_oP|m$436EYWNjj;sCQ8qM^qi`Vm-sNyrtcvqIZ_tftZ9K1Z% zq@6O9BEIGlk$28Q2JMd5BiL}p0T`eP@c*V_z$;*6D!rpw=m4q`Zi<#9NSEPfEWn8h zu2jjc{~Ed)E+!!$zgioSD5965;xxGnfW60kM2}(RudU9+0NeoAOimUrq}oxf#nCA%7g~+1aUScNwa)@jojWe$t#Xu^mhDp33Vve%X}twkl9sR? zmhkP3K%ENFN#flel9^C5-shr~ghuZ%XdLLH&6{+Csz32LzufUvfBdQ7vsje!bXH6T zcmCWfo3h+Z-W4no0^6OPc7X45SY%W7cI_J`a7QNei>3-&=1TRzl&=bn4jSddG@c%Y z?icqzsC|q)FmIH+x>xz0_NP52h+wm)##@{Wp?lS9B6_!8>U^yuxa&xg1;)=-SXSrS zet(Y)J@Z~IFH1Q%S$~FaAX0Nk7J(N39%3}c+N(q+ozwpxr&@Z>pL_m;0UR!uZ^yEW zp(1K*u`hg*ef0N0GI6KuvSasEhw}2J>Q;9!4;T0X^0weBr4i}%^KQ4g(g5)Xj`>?o z`O~8NO3S*qP55}hl_+3f@z>Unx!kvKuIl@rsZXMoPX&4$Yr+`nl7N-8zndog+Y>Pa z9HOdHU;D1ZsZ`cG#sBM%Z=B}_69A@^Uu9i{eI;t~HqlDPJ8TyHMYQ*)X|xr7KnJeF z0W3U^6#KfDL&-|FboJ$G^`Pr^a*}Scv)!4DOwutHZ$J#wpJzVv{>$D6kAUCwKE614 z-$7~j2}~R%Z`kTc08-?Or7Ewfj%fKX2)p0HDUJ3ThmPD=9MD06z^be@w|bK`*F1Pf zm}_E>69rKKL5XR$xY-cRCC!Z_;KMUum9k6yBiFpi`kO^38fwrYv+(AYTIO!@{2fxJ zaw8iD;uu*K%gUdaQIHs>cIF}ByyH}UhL$(PBYulX{h{8OEE1dL@z-q)qidh~U2%10 zu1Uo%vb9Bia11>o3#F)W#W$w7N^L!x_S>E%Z`S#& z!cJJu8VTg8TykWx?JMa-ZFn09P%6bojY~VNw*`yCe|K@FU<%*tO^cfbHTS0g2YPeg z``U6}SA|Abw6UtCy-=mo5eZRH8Rq3ioH?2!HNQMphH3&#$og#r)|ym;9_yIr`@YP& z3nc`ht6gidtKBa><^425V3HM_3EICaMx#j}eOBs>*q%gQFDL#E@JtAg)%hZ0SfUZUZx$b!Zo0B1W zm)Np3HRpJGBFa#1KoefYLt>NnQf)lb0DqJydId9Y)l})wku}hHARg0{uB!Y-ys5pA z!loKVJ5W(8u!~F$-oPVFVyV)W>~X7Gk#I^pF(7xCfW=>8HwP9U-F7IFobZKloyN@7 zvaY#Kh?Oo8Wrx)~6j=Kz-u}}j0+h@*ZgaiOqMoxZ2U4I1NPLAOI-2WaO&{1YSy@L1m7_Q((btw1YPdE(N@(Qi< zSFIAU=#SEqp62`;5BOvq7nngv^>1#^h0dv9HEZ}e1}+^a6^>f+*A(n zwLLmXUeL{ZLAD8TD!o&g~4~l#N~EDaYO!| zlXWBbAk_sPx>Krr*|A>L<*h-@Sn|cPp4O}ISK5!j)p9c|bWXAqull%_HxP=jpEh4O zCj@X7xWdFvD{|yZd-IYe>PECcs#@fNRpaH6lh!Xz@h9rA;C3UZP({Ktjf-8Icwen5 zeDPvxC@||l{VzyD`X}fAU~w^~O?=^E9&^n&s6-@_K7EicsYUXywwj^zMkyr%%45TK zX6%JMJ2tESD_ClvV3Y}wz}&TIC2jbISeC1sN_r4?l;trtNf5~J)y0EYlHQaG6P|)P zPvCV6L7)CRJyBPyG=+(lhW%JTMD;N`!R5rbE02kk-l;wf9JdLCETCXhc5Kp>_i1Ew zRd7<^3pXjqC5DQ}=L^ZE2e>eqes6)GTjOx)hf?(0sGu)qNhdxGigBFt`i7iqM>hz8 z5f@UBa21fk#lzklSt=q`i94o1v&~zp^gRolOktB?k1pY7X|3l;Y0HV~g%k)VLvev^ ziPIm=S?NYC67f*25O!$}dqJ1YDYQ|t#R4TJV`0MaK1OTVzAI*Ot$!?BFj;5srut8q z?;>}OmfuL&EVDljb*dqnGk<6TykGwDe9g5{*G`o_m??8w}|-z}h`t&0H&0W9)ZSmUSDHX7B} zR%K6Np<;G53Q$Qkh0mnN!_Ov&^=hYOzJ#dyI%JjL3}#fK=qDiXV)ZX62x@yr#Z%H; z-zui^n-@A>hj|sjomybl5Wi5;7RZ%4R+!a5dN!$4Jou(or1Xx{rOqod5jZi50}}8SDxS#P z^kLoPUn=2FD1*_OMHTLm0r6Qbuh`70?ZG~b8wg>5qxufMSMK9P!RNJ3Sqm&;?@meQ zmza9%NtR$RG~3;?cOAv8!C=9#vY|gKBpGX(g2(>44V;*S`Q%lTb}DH>#Xi_%yqZUJ zkx-st+j{*Dj{@9M95NON1T@$wspGBdfYNoyO0{W)l1Xl%g5QyX^V54n rfBQ>gnYI~pFzN1WX(M?7fTt&L2ZcxAORgBysU<*DRZpc(2@?4~yZF-7 literal 0 HcmV?d00001 diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts new file mode 100644 index 00000000..b57c58e3 --- /dev/null +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -0,0 +1,17 @@ +import { setResult, TaskResult } from "azure-pipelines-task-lib/task" +import { debug, warning, error } from "azure-pipelines-task-lib/task" + +async function run() { + try { + + // TODO: This... + setResult(TaskResult.Succeeded); + + } + catch (e: any) { + error(`Unhandled exception: ${e}`); + setResult(TaskResult.Failed, e?.message); + } +} + +run(); diff --git a/extension/tasks/dependabot/dependabotV2/task.json b/extension/tasks/dependabot/dependabotV2/task.json new file mode 100644 index 00000000..fd494c4b --- /dev/null +++ b/extension/tasks/dependabot/dependabotV2/task.json @@ -0,0 +1,262 @@ +{ + "$schema": "https://raw.githubusercontent.com/Microsoft/azure-pipelines-task-lib/master/tasks.schema.json", + "id": "d98b873d-cf18-41eb-8ff5-234f14697896", + "name": "dependabot", + "friendlyName": "Dependabot", + "description": "Automatically update dependencies and vulnerabilities in your code", + "helpMarkDown": "For help please visit https://github.com/tinglesoftware/dependabot-azure-devops/issues", + "helpUrl": "https://github.com/tinglesoftware/dependabot-azure-devops/issues", + "releaseNotes": "https://github.com/tinglesoftware/dependabot-azure-devops/releases", + "category": "Utility", + "visibility": ["Build", "Release"], + "runsOn": ["Agent", "DeploymentGroup"], + "author": "Tingle Software", + "demands": [], + "version": { + "Major": 2, + "Minor": 0, + "Patch": 0 + }, + "instanceNameFormat": "Dependabot", + "minimumAgentVersion": "3.232.1", + "groups": [ + { + "name": "security_updates", + "displayName": "Security advisories and vulnerabilities", + "isExpanded": false + }, + { + "name": "pull_requests", + "displayName": "Pull request options", + "isExpanded": false + }, + { + "name": "devops", + "displayName": "Azure DevOps authentication", + "isExpanded": false + }, + { + "name": "github", + "displayName": "GitHub authentication", + "isExpanded": false + }, + { + "name": "advanced", + "displayName": "Advanced", + "isExpanded": false + } + ], + "inputs": [ + { + "name": "useUpdateScriptvNext", + "type": "boolean", + "groupName": "advanced", + "label": "Use latest update script (vNext) (Experimental)", + "defaultValue": "false", + "required": false, + "helpMarkDown": "Determines if the task will use the newest 'vNext' update script instead of the default update script. This Defaults to `false`. See the [vNext update script documentation](https://github.com/tinglesoftware/dependabot-azure-devops/pull/1186) for more information." + }, + + { + "name": "failOnException", + "type": "boolean", + "groupName": "advanced", + "label": "Fail task when an update exception occurs.", + "defaultValue": true, + "required": false, + "helpMarkDown": "When set to `true`, a failure in updating a single dependency will cause the container execution to fail thereby causing the task to fail. This is important when you want a single failure to prevent trying to update other dependencies." + }, + + { + "name": "skipPullRequests", + "type": "boolean", + "groupName": "pull_requests", + "label": "Skip creation and updating of pull requests.", + "defaultValue": false, + "required": false, + "helpMarkDown": "When set to `true` the logic to update the dependencies is executed but the actual Pull Requests are not created/updated. Defaults to `false`." + }, + { + "name": "commentPullRequests", + "type": "boolean", + "groupName": "pull_requests", + "label": "Comment on abandoned pull requests with close reason.", + "defaultValue": false, + "required": false, + "helpMarkDown": "When set to `true` a comment will be added to abandoned pull requests explanating why it was closed. Defaults to `false`." + }, + { + "name": "abandonUnwantedPullRequests", + "type": "boolean", + "groupName": "pull_requests", + "label": "Abandon unwanted pull requests.", + "defaultValue": false, + "required": false, + "helpMarkDown": "When set to `true` pull requests that are no longer needed are closed at the tail end of the execution. Defaults to `false`." + }, + { + "name": "setAutoComplete", + "type": "boolean", + "groupName": "pull_requests", + "label": "Auto-complete pull requests when all policies pass", + "defaultValue": false, + "required": false, + "helpMarkDown": "When set to `true`, pull requests that pass all policies will be merged automatically. Defaults to `false`." + }, + { + "name": "mergeStrategy", + "type": "pickList", + "groupName": "pull_requests", + "label": "Merge Strategy", + "defaultValue": "squash", + "required": true, + "helpMarkDown": "The merge strategy to use. Learn more [here](https://learn.microsoft.com/en-us/rest/api/azure/devops/git/pull-requests/update?view=azure-devops-rest-5.1&tabs=HTTP#gitpullrequestmergestrategy).", + "options": { + "noFastForward": "No fast forward", + "rebase": "Rebase", + "rebaseMerge": "Rebase merge", + "squash": "Squash" + }, + "visibleRule": "setAutoComplete=true" + }, + { + "name": "autoCompleteIgnoreConfigIds", + "type": "string", + "groupName": "pull_requests", + "label": "Semicolon delimited list of any policy configuration IDs which auto-complete should not wait for.", + "defaultValue": "", + "required": false, + "helpMarkDown": "A semicolon (`;`) delimited list of any policy configuration IDs which auto-complete should not wait for. Only applies to optional policies (isBlocking == false). Auto-complete always waits for required policies (isBlocking == true).", + "visibleRule": "setAutoComplete=true" + }, + { + "name": "autoApprove", + "type": "boolean", + "groupName": "pull_requests", + "label": "Auto-approve pull requests", + "defaultValue": false, + "required": false, + "helpMarkDown": "When set to `true`, pull requests will automatically be approved by the specified user. Defaults to `false`." + }, + { + "name": "autoApproveUserToken", + "type": "string", + "groupName": "pull_requests", + "label": "A personal access token of the user that should approve the PR.", + "defaultValue": "", + "required": false, + "helpMarkDown": "A personal access token of the user of that shall be used to approve the created PR automatically. If the same user that creates the PR should approve, this can be left empty. This won't work with if the Build Service with the build service account!", + "visibleRule": "autoApprove=true" + }, + + { + "name": "gitHubConnection", + "type": "connectedService:github:OAuth,PersonalAccessToken,InstallationToken,Token", + "groupName": "github", + "label": "GitHub connection (OAuth or PAT)", + "defaultValue": "", + "required": false, + "helpMarkDown": "Specify the name of the GitHub service connection to use to connect to the GitHub repositories. The connection must be based on a GitHub user's OAuth or a GitHub personal access token. Learn more about service connections [here](https://aka.ms/AA3am5s)." + }, + { + "name": "gitHubAccessToken", + "type": "string", + "groupName": "github", + "label": "GitHub Personal Access Token.", + "defaultValue": "", + "required": false, + "helpMarkDown": "The raw Personal Access Token for accessing GitHub repositories. Use this in place of `gitHubConnection` such as when it is not possible to create a service connection." + }, + + { + "name": "securityAdvisoriesFile", + "type": "string", + "label": "Path for the file containing security advisories in JSON format.", + "groupName": "security_updates", + "helpMarkDown": "The file containing security advisories.", + "required": false + }, + + { + "name": "azureDevOpsServiceConnection", + "type": "connectedService:Externaltfs", + "groupName": "devops", + "label": "Azure DevOps Service Connection to use.", + "required": false, + "helpMarkDown": "Specify a service connection to use, if you want to use a different service principal than the default to create your PRs." + }, + { + "name": "azureDevOpsAccessToken", + "type": "string", + "groupName": "devops", + "label": "Azure DevOps Personal Access Token.", + "required": false, + "helpMarkDown": "The Personal Access Token for accessing Azure DevOps repositories. Supply a value here to avoid using permissions for the Build Service either because you cannot change its permissions or because you prefer that the Pull Requests be done by a different user. Use this in place of `azureDevOpsServiceConnection` such as when it is not possible to create a service connection." + }, + { + "name": "targetRepositoryName", + "type": "string", + "groupName": "advanced", + "label": "Target Repository Name", + "required": false, + "helpMarkDown": "The name of the repository to target for processing. If this value is not supplied then the Build Repository Name is used. Supplying this value allows creation of a single pipeline that runs Dependabot against multiple repositories." + }, + { + "name": "targetUpdateIds", + "type": "string", + "groupName": "advanced", + "label": "Semicolon delimited list of update identifiers to run.", + "defaultValue": "", + "required": false, + "helpMarkDown": "A semicolon (`;`) delimited list of update identifiers run. Index are zero-based and in the order written in the configuration file. When not present, all the updates are run. This is meant to be used in scenarios where you want to run updates a different times from the same configuration file given you cannot schedule them independently in the pipeline." + }, + { + "name": "updaterOptions", + "type": "string", + "groupName": "advanced", + "label": "Comma separated list of Dependabot experiments (updater options).", + "required": false, + "helpMarkDown": "Set a list of Dependabot experiments (updater options) in CSV format. Available options depend on the ecosystem. Example: `goprivate=true,kubernetes_updates=true`." + }, + { + "name": "excludeRequirementsToUnlock", + "type": "string", + "groupName": "advanced", + "label": "Space-separated list of dependency updates requirements to be excluded.", + "required": false, + "helpMarkDown": "Exclude certain dependency updates requirements. See list of allowed values [here](https://github.com/dependabot/dependabot-core/issues/600#issuecomment-407808103). Useful if you have lots of dependencies and the update script too slow. The values provided are space-separated. Example: `own all` to only use the `none` version requirement." + }, + { + "name": "dockerImageTag", + "type": "string", + "groupName": "advanced", + "label": "Tag of the docker image to be pulled.", + "required": false, + "helpMarkDown": "The image tag to use when pulling the docker container used by the task. A tag also defines the version. By default, the task decides which tag/version to use. This can be the latest or most stable version. You can also use `major.minor` format to get the latest patch" + }, + { + "name": "extraEnvironmentVariables", + "type": "string", + "groupName": "advanced", + "label": "Semicolon delimited list of environment variables", + "required": false, + "defaultValue": "", + "helpMarkDown": "A semicolon (`;`) delimited list of environment variables that are sent to the docker container. See possible use case [here](https://github.com/tinglesoftware/dependabot-azure-devops/issues/138)" + }, + { + "name": "forwardHostSshSocket", + "type": "boolean", + "groupName": "advanced", + "label": "Forward the host ssh socket", + "defaultValue": "false", + "required": false, + "helpMarkDown": "Ensure that the host ssh socket is forwarded to the container to authenticate with ssh" + } + ], + "dataSourceBindings": [], + "execution": { + "Node20_1": { + "target": "index.js" + } + } +} diff --git a/extension/task/IDependabotConfig.ts b/extension/tasks/utils/IDependabotConfig.ts similarity index 100% rename from extension/task/IDependabotConfig.ts rename to extension/tasks/utils/IDependabotConfig.ts diff --git a/extension/task/utils/convertPlaceholder.ts b/extension/tasks/utils/convertPlaceholder.ts similarity index 100% rename from extension/task/utils/convertPlaceholder.ts rename to extension/tasks/utils/convertPlaceholder.ts diff --git a/extension/task/utils/extractHostname.ts b/extension/tasks/utils/extractHostname.ts similarity index 100% rename from extension/task/utils/extractHostname.ts rename to extension/tasks/utils/extractHostname.ts diff --git a/extension/task/utils/extractOrganization.ts b/extension/tasks/utils/extractOrganization.ts similarity index 100% rename from extension/task/utils/extractOrganization.ts rename to extension/tasks/utils/extractOrganization.ts diff --git a/extension/task/utils/extractVirtualDirectory.ts b/extension/tasks/utils/extractVirtualDirectory.ts similarity index 100% rename from extension/task/utils/extractVirtualDirectory.ts rename to extension/tasks/utils/extractVirtualDirectory.ts diff --git a/extension/task/utils/getAzureDevOpsAccessToken.ts b/extension/tasks/utils/getAzureDevOpsAccessToken.ts similarity index 100% rename from extension/task/utils/getAzureDevOpsAccessToken.ts rename to extension/tasks/utils/getAzureDevOpsAccessToken.ts diff --git a/extension/task/utils/getDockerImageTag.ts b/extension/tasks/utils/getDockerImageTag.ts similarity index 100% rename from extension/task/utils/getDockerImageTag.ts rename to extension/tasks/utils/getDockerImageTag.ts diff --git a/extension/task/utils/getGithubAccessToken.ts b/extension/tasks/utils/getGithubAccessToken.ts similarity index 100% rename from extension/task/utils/getGithubAccessToken.ts rename to extension/tasks/utils/getGithubAccessToken.ts diff --git a/extension/task/utils/getSharedVariables.ts b/extension/tasks/utils/getSharedVariables.ts similarity index 100% rename from extension/task/utils/getSharedVariables.ts rename to extension/tasks/utils/getSharedVariables.ts diff --git a/extension/task/utils/parseConfigFile.ts b/extension/tasks/utils/parseConfigFile.ts similarity index 99% rename from extension/task/utils/parseConfigFile.ts rename to extension/tasks/utils/parseConfigFile.ts index e158f679..356aa7f5 100644 --- a/extension/task/utils/parseConfigFile.ts +++ b/extension/tasks/utils/parseConfigFile.ts @@ -5,7 +5,7 @@ import * as fs from 'fs'; import { load } from 'js-yaml'; import * as path from 'path'; import { URL } from 'url'; -import { IDependabotConfig, IDependabotRegistry, IDependabotUpdate } from '../IDependabotConfig'; +import { IDependabotConfig, IDependabotRegistry, IDependabotUpdate } from './IDependabotConfig'; import { convertPlaceholder } from './convertPlaceholder'; import { ISharedVariables } from './getSharedVariables'; diff --git a/extension/task/utils/resolveAzureDevOpsIdentities.ts b/extension/tasks/utils/resolveAzureDevOpsIdentities.ts similarity index 100% rename from extension/task/utils/resolveAzureDevOpsIdentities.ts rename to extension/tasks/utils/resolveAzureDevOpsIdentities.ts diff --git a/extension/tests/utils/convertPlaceholder.test.ts b/extension/tests/utils/convertPlaceholder.test.ts index eed88400..f63dfd0b 100644 --- a/extension/tests/utils/convertPlaceholder.test.ts +++ b/extension/tests/utils/convertPlaceholder.test.ts @@ -1,4 +1,4 @@ -import { extractPlaceholder } from '../../task/utils/convertPlaceholder'; +import { extractPlaceholder } from '../../tasks/utils/convertPlaceholder'; describe('Parse property placeholder', () => { it('Should return key with underscores', () => { diff --git a/extension/tests/utils/extractHostname.test.ts b/extension/tests/utils/extractHostname.test.ts index ca8d01f7..00570b41 100644 --- a/extension/tests/utils/extractHostname.test.ts +++ b/extension/tests/utils/extractHostname.test.ts @@ -1,4 +1,4 @@ -import extractHostname from '../../task/utils/extractHostname'; +import extractHostname from '../../tasks/utils/extractHostname'; describe('Extract hostname', () => { it('Should convert old *.visualstudio.com hostname to dev.azure.com', () => { diff --git a/extension/tests/utils/extractOrganization.test.ts b/extension/tests/utils/extractOrganization.test.ts index a827bae0..738201ae 100644 --- a/extension/tests/utils/extractOrganization.test.ts +++ b/extension/tests/utils/extractOrganization.test.ts @@ -1,4 +1,4 @@ -import extractOrganization from '../../task/utils/extractOrganization'; +import extractOrganization from '../../tasks/utils/extractOrganization'; describe('Extract organization name', () => { it('Should extract organization for on-premise domain', () => { diff --git a/extension/tests/utils/extractVirtualDirectory.test.ts b/extension/tests/utils/extractVirtualDirectory.test.ts index 35a734b8..ead119f5 100644 --- a/extension/tests/utils/extractVirtualDirectory.test.ts +++ b/extension/tests/utils/extractVirtualDirectory.test.ts @@ -1,4 +1,4 @@ -import extractVirtualDirectory from '../../task/utils/extractVirtualDirectory'; +import extractVirtualDirectory from '../../tasks/utils/extractVirtualDirectory'; describe('Extract virtual directory', () => { it('Should extract virtual directory', () => { diff --git a/extension/tests/utils/parseConfigFile.test.ts b/extension/tests/utils/parseConfigFile.test.ts index 864218a2..a97a9ffe 100644 --- a/extension/tests/utils/parseConfigFile.test.ts +++ b/extension/tests/utils/parseConfigFile.test.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import { load } from 'js-yaml'; -import { IDependabotRegistry, IDependabotUpdate } from '../../task/IDependabotConfig'; -import { parseRegistries, parseUpdates, validateConfiguration } from '../../task/utils/parseConfigFile'; +import { IDependabotRegistry, IDependabotUpdate } from '../../tasks/utils/IDependabotConfig'; +import { parseRegistries, parseUpdates, validateConfiguration } from '../../tasks/utils/parseConfigFile'; describe('Parse configuration file', () => { it('Parsing works as expected', () => { diff --git a/extension/tests/utils/resolveAzureDevOpsIdentities.test.ts b/extension/tests/utils/resolveAzureDevOpsIdentities.test.ts index d3e54d4c..87953134 100644 --- a/extension/tests/utils/resolveAzureDevOpsIdentities.test.ts +++ b/extension/tests/utils/resolveAzureDevOpsIdentities.test.ts @@ -1,6 +1,6 @@ import axios from 'axios'; import { describe } from 'node:test'; -import { isHostedAzureDevOps, resolveAzureDevOpsIdentities } from '../../task/utils/resolveAzureDevOpsIdentities'; +import { isHostedAzureDevOps, resolveAzureDevOpsIdentities } from '../../tasks/utils/resolveAzureDevOpsIdentities'; describe('isHostedAzureDevOps', () => { it('Old visualstudio url is hosted.', () => { diff --git a/extension/vss-extension.json b/extension/vss-extension.json index 7be9b6ec..c7f1eb70 100644 --- a/extension/vss-extension.json +++ b/extension/vss-extension.json @@ -31,7 +31,7 @@ }, "files": [ { - "path": "task" + "path": "tasks" }, { "path": "images", @@ -44,7 +44,7 @@ "type": "ms.vss-distributed-task.task", "targets": ["ms.vss-distributed-task.tasks"], "properties": { - "name": "task" + "name": "tasks/dependabot" } } ] From eae688c261669b70ae7e01a8981ac02ae7e6f09b Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Mon, 2 Sep 2024 01:57:34 +1200 Subject: [PATCH 02/57] Basic support for running update using dependabot-cli --- docs/extension.md | 8 +- extension/package.json | 2 + .../tasks/dependabot/dependabotV2/index.ts | 155 ++++++++++++++++- extension/tasks/utils/dependabotUpdater.ts | 162 ++++++++++++++++++ 4 files changed, 320 insertions(+), 7 deletions(-) create mode 100644 extension/tasks/utils/dependabotUpdater.ts diff --git a/docs/extension.md b/docs/extension.md index ef3b5061..08530340 100644 --- a/docs/extension.md +++ b/docs/extension.md @@ -28,7 +28,7 @@ npm install ```bash cd extension -npm run build:prod +npm run build ``` To generate the Azure DevOps `.vsix` extension package for testing, you'll first need to [create a publisher account](https://learn.microsoft.com/en-us/azure/devops/extend/publish/overview?view=azure-devops#create-a-publisher) on the [Visual Studio Marketplace Publishing Portal](https://marketplace.visualstudio.com/manage/createpublisher?managePageRedirect=true). After this, override your publisher ID below and generate the extension with: @@ -41,6 +41,12 @@ npm run package -- --overrides-file overrides.local.json --override "{\"publishe To test the extension in Azure DevOps, you'll first need to build the extension `.vsix` file (see above). After this, [publish your extension](https://learn.microsoft.com/en-us/azure/devops/extend/publish/overview?view=azure-devops#publish-your-extension), then [install your extension](https://learn.microsoft.com/en-us/azure/devops/extend/publish/overview?view=azure-devops#install-your-extension). +## Running the task locally + +```bash +npm run task:V2 +``` + ## Running the unit tests ```bash diff --git a/extension/package.json b/extension/package.json index a6094861..577fe195 100644 --- a/extension/package.json +++ b/extension/package.json @@ -6,6 +6,8 @@ "scripts": { "build": "cp -r node_modules tasks/dependabot/dependabotV1/node_modules && cp -r node_modules tasks/dependabot/dependabotV2/node_modules && tsc -p .", "package": "npx tfx-cli extension create --json5", + "task:V1": "node tasks/dependabot/dependabotV1/index.js", + "task:V2": "node tasks/dependabot/dependabotV2/index.js", "test": "jest" }, "repository": { diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index b57c58e3..961a21b6 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -1,17 +1,160 @@ -import { setResult, TaskResult } from "azure-pipelines-task-lib/task" +import { which, setResult, TaskResult } from "azure-pipelines-task-lib/task" import { debug, warning, error } from "azure-pipelines-task-lib/task" +import { DependabotUpdater, IUpdateScenarioOutput } from '../../utils/dependabotUpdater'; +import { parseConfigFile } from '../../utils/parseConfigFile'; +import getSharedVariables from '../../utils/getSharedVariables'; async function run() { + let updater: DependabotUpdater = undefined; try { - - // TODO: This... - setResult(TaskResult.Succeeded); + + // Check if required tools are installed + debug('Checking for `docker` install...'); + which('docker', true); + debug('Checking for `go` install...'); + which('go', true); + + // Parse the dependabot configuration file + const variables = getSharedVariables(); + const config = await parseConfigFile(variables); + + // Initialise the dependabot updater + updater = new DependabotUpdater(undefined /* TODO: Add config for this */, variables.debug); + + // Process the updates per-ecosystem + let updatedSuccessfully: boolean = true; + config.updates.forEach(async (update) => { + + // TODO: Fetch all existing PRs from DevOps + + let extraCredentials = new Array(); + for (const key in config.registries) { + const registry = config.registries[key]; + extraCredentials.push({ + type: registry.type, + host: registry.host, + url: registry.url, + registry: registry.registry, + username: registry.username, + password: registry.password, + token: registry.token + }); + }; + + // Run dependabot updater for the job + const result = processUpdateOutputs( + await updater.update({ + // TODO: Parse this from `config` and `variables` + job: { + id: 'job-1', + job: { + 'package-manager': update.packageEcosystem, + 'allowed-updates': [ + { 'update-type': 'all' } + ], + source: { + provider: 'azure', + repo: `${variables.organization}/${variables.project}/_git/${variables.repository}`, + directory: update.directory, + commit: undefined + } + }, + credentials: (extraCredentials || []).concat([ + { + type: 'git_source', + host: new URL(variables.organizationUrl).hostname, + username: 'x-access-token', + password: variables.systemAccessToken + } + ]) + } + }) + ); + if (!result) { + updatedSuccessfully = false; + } + + // TODO: Loop through all existing PRs and do a single update job for each, update/close the PR as needed + + }); + + setResult( + updatedSuccessfully ? TaskResult.Succeeded : TaskResult.Failed, + updatedSuccessfully ? 'All update jobs completed successfully' : 'One or more update jobs failed, check logs for more information' + ); } catch (e: any) { - error(`Unhandled exception: ${e}`); - setResult(TaskResult.Failed, e?.message); + error(`Unhandled task exception: ${e}`); + console.log(e); + setResult(TaskResult.Failed, e?.message); } + finally { + updater?.cleanup(); + } +} + +// Process the job outputs and apply changes to DevOps +// TODO: Move this to a new util class, e.g. `dependabotOutputProcessor.ts` +function processUpdateOutputs(outputs: IUpdateScenarioOutput[]) : boolean { + let success: boolean = true; + outputs.forEach(output => { + switch (output.type) { + + case 'update_dependency_list': + console.log('TODO: UPDATED DEPENDENCY LIST: ', output.data); + // TODO: Save data to DevOps? This would be really useful for generating a dependency graph hub page or HTML report (future feature maybe?) + break; + + case 'create_pull_request': + console.log('TODO: CREATE PULL REQUEST: ', output.data); + // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: create_pull_request() + break; + + case 'update_pull_request': + console.log('TODO: UPDATE PULL REQUEST ', output.data); + // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: update_pull_request() + break; + + case 'close_pull_request': + console.log('TODO: CLOSE PULL REQUEST ', output.data); + // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: close_pull_request() + break; + + case 'mark_as_processed': + console.log('TODO: MARK AS PROCESSED: ', output.data); + // TODO: Log info? + break; + + case 'record_ecosystem_versions': + console.log('TODO: RECORD ECOSYSTEM VERSIONS: ', output.data); + // TODO: Log info? + break; + + case 'record_update_job_error': + console.log('TODO: RECORD UPDATE JOB ERROR: ', output.data); + // TODO: Log error? + success = false; + break; + + case 'record_update_job_unknown_error': + console.log('TODO: RECORD UPDATE JOB UNKNOWN ERROR: ', output.data); + // TODO: Log error? + success = false; + break; + + case 'increment_metric': + console.log('TODO: INCREMENT METRIC: ', output.data); + // TODO: Log info? + break; + + default: + warning(`Unknown dependabot output type: ${output.type}`); + success = false; + break; + } + }); + return success; } run(); diff --git a/extension/tasks/utils/dependabotUpdater.ts b/extension/tasks/utils/dependabotUpdater.ts new file mode 100644 index 00000000..1b37a736 --- /dev/null +++ b/extension/tasks/utils/dependabotUpdater.ts @@ -0,0 +1,162 @@ +import { debug, warning, error } from "azure-pipelines-task-lib/task" +import { which, tool } from "azure-pipelines-task-lib/task" +import { ToolRunner } from "azure-pipelines-task-lib/toolrunner" +import * as yaml from 'js-yaml'; +import * as path from 'path'; +import * as fs from 'fs'; +import * as os from 'os'; + +export interface IUpdateJobConfig { + id: string, + job: { + 'package-manager': string, + 'allowed-updates': { + 'update-type': string + }[], + source: { + provider: string, + repo: string, + directory: string, + commit: string + } + }, + credentials: { + type: string, + host?: string, + username?: string, + password?: string, + url?: string, + token?: string + }[] +} + +export interface IUpdateScenarioOutput { + type: string, + data: any +} + +export class DependabotUpdater { + private readonly jobsPath: string; + private readonly toolImage: string; + private readonly debug: boolean; + + constructor(cliToolImage?: string, debug?: boolean) { + this.jobsPath = path.join(os.tmpdir(), 'dependabot-jobs'); + this.toolImage = cliToolImage ?? "github.com/dependabot/cli/cmd/dependabot@latest"; + this.debug = debug ?? false; + this.ensureJobsPathExists(); + } + + // Run dependabot update + public async update(options: { + job: IUpdateJobConfig, + collectorImage?: string, + proxyImage?: string, + updaterImage?: string + }): Promise { + + // Install dependabot if not already installed + await this.ensureToolsAreInstalled(); + + // Create the job directory + const jobId = options.job.id; + const jobPath = path.join(this.jobsPath, jobId.toString()); + const jobInputPath = path.join(jobPath, 'job.yaml'); + const jobOutputPath = path.join(jobPath, 'scenario.yaml'); + this.ensureJobsPathExists(); + if (!fs.existsSync(jobPath)){ + fs.mkdirSync(jobPath); + } + + // Generate the job input file + writeJobInput(jobInputPath, options.job); + + // Compile dependabot cmd arguments + let dependabotArguments = [ + "update", "-f", jobInputPath, "-o", jobOutputPath + ]; + if (options.collectorImage) { + dependabotArguments.push("--collector-image", options.collectorImage); + } + if (options.proxyImage) { + dependabotArguments.push("--proxy-image", options.proxyImage); + } + if (options.updaterImage) { + dependabotArguments.push("--updater-image", options.updaterImage); + } + + console.info("Running dependabot update..."); + const dependabotTool = tool(which("dependabot", true)).arg(dependabotArguments); + const dependabotResultCode = await dependabotTool.execAsync({ + silent: !this.debug + }); + if (dependabotResultCode != 0) { + throw new Error(`Dependabot update failed with exit code ${dependabotResultCode}`); + } + + return readScenarioOutputs(jobOutputPath); + } + + // Install dependabot if not already installed + private async ensureToolsAreInstalled(): Promise { + + debug('Checking for `dependabot` install...'); + if (which("dependabot", false)) { + return; + } + + debug("Dependabot CLI was not found, installing with `go install`..."); + const goTool: ToolRunner = tool(which("go", true)); + goTool.arg(["install", this.toolImage]); + goTool.execSync({ + silent: !this.debug + }); + } + + // Create the jobs directory if it does not exist + private ensureJobsPathExists(): void { + if (!fs.existsSync(this.jobsPath)){ + fs.mkdirSync(this.jobsPath); + } + } + + // Clean up the jobs directory and its contents + public cleanup(): void { + if (fs.existsSync(this.jobsPath)){ + fs.rmSync(this.jobsPath, { + recursive: true, + force: true + }); + } + } +} + +function writeJobInput(path: string, job: IUpdateJobConfig): void { + fs.writeFileSync(path, yaml.dump(job)); +} + +function readScenarioOutputs(scenarioFilePath: string): IUpdateScenarioOutput[] { + if (!scenarioFilePath) { + throw new Error("Scenario file path is required"); + } + + const scenarioContent = fs.readFileSync(scenarioFilePath, 'utf-8'); + if (!scenarioContent || typeof scenarioContent !== 'string') { + throw new Error(`Scenario file could not be read at '${scenarioFilePath}'`); + } + + const scenario: any = yaml.load(scenarioContent); + if (scenario === null || typeof scenario !== 'object') { + throw new Error('Invalid scenario object'); + } + + let outputs = new Array(); + scenario['output']?.forEach((output: any) => { + outputs.push({ + type: output['type'], + data: output['expect']?.['data'] + }); + }); + + return outputs; +} \ No newline at end of file From 83bd07a25246ab6481a981484e7c0ffb4b999de3 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Mon, 2 Sep 2024 15:48:47 +1200 Subject: [PATCH 03/57] Add missing update job configs --- .../tasks/dependabot/dependabotV2/index.ts | 78 +++++----- extension/tasks/utils/dependabotUpdater.ts | 138 ++++++++++++++---- extension/tasks/utils/getSharedVariables.ts | 10 +- 3 files changed, 157 insertions(+), 69 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index 961a21b6..51347be9 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -1,6 +1,6 @@ import { which, setResult, TaskResult } from "azure-pipelines-task-lib/task" import { debug, warning, error } from "azure-pipelines-task-lib/task" -import { DependabotUpdater, IUpdateScenarioOutput } from '../../utils/dependabotUpdater'; +import { DependabotUpdater, IUpdateJobConfig, IUpdateScenarioOutput } from '../../utils/dependabotUpdater'; import { parseConfigFile } from '../../utils/parseConfigFile'; import getSharedVariables from '../../utils/getSharedVariables'; @@ -19,7 +19,8 @@ async function run() { const config = await parseConfigFile(variables); // Initialise the dependabot updater - updater = new DependabotUpdater(undefined /* TODO: Add config for this */, variables.debug); + // TODO: Add config for CLI image argument + updater = new DependabotUpdater(null, variables.debug); // Process the updates per-ecosystem let updatedSuccessfully: boolean = true; @@ -27,54 +28,60 @@ async function run() { // TODO: Fetch all existing PRs from DevOps - let extraCredentials = new Array(); + let registryCredentials = new Array(); for (const key in config.registries) { const registry = config.registries[key]; - extraCredentials.push({ + registryCredentials.push({ type: registry.type, host: registry.host, + region: undefined, // TODO: registry.region, url: registry.url, registry: registry.registry, username: registry.username, password: registry.password, - token: registry.token + token: registry.token, + 'replaces-base': registry['replaces-base'] }); }; - // Run dependabot updater for the job - const result = processUpdateOutputs( - await updater.update({ - // TODO: Parse this from `config` and `variables` - job: { - id: 'job-1', - job: { - 'package-manager': update.packageEcosystem, - 'allowed-updates': [ - { 'update-type': 'all' } - ], - source: { - provider: 'azure', - repo: `${variables.organization}/${variables.project}/_git/${variables.repository}`, - directory: update.directory, - commit: undefined - } - }, - credentials: (extraCredentials || []).concat([ - { - type: 'git_source', - host: new URL(variables.organizationUrl).hostname, - username: 'x-access-token', - password: variables.systemAccessToken - } - ]) + let job: IUpdateJobConfig = { + job: { + // TODO: Parse all options from `config` and `variables` + id: 'job-1', + 'package-manager': update.packageEcosystem, + 'updating-a-pull-request': false, + 'allowed-updates': [ + { 'update-type': 'all' } + ], + 'security-updates-only': false, + source: { + provider: 'azure', + 'api-endpoint': variables.apiEndpointUrl, + hostname: variables.hostname, + repo: `${variables.organization}/${variables.project}/_git/${variables.repository}`, + branch: update.targetBranch, // TODO: add config for 'source branch'?? + commit: undefined, // TODO: add config for this? + directory: update.directories?.length == 0 ? update.directory : undefined, + directories: update.directories?.length > 0 ? update.directories : undefined + } + }, + credentials: (registryCredentials || []).concat([ + { + type: 'git_source', + host: variables.hostname, + username: variables.systemAccessUser?.trim()?.length > 0 ? variables.systemAccessUser : 'x-access-token', + password: variables.systemAccessToken } - }) - ); - if (!result) { + ]) + }; + + // Run dependabot updater for the job + if (!processUpdateOutputs(await updater.update(job))) { updatedSuccessfully = false; } // TODO: Loop through all existing PRs and do a single update job for each, update/close the PR as needed + // e.g. https://github.com/dependabot/cli/blob/main/testdata/go/update-pr.yaml }); @@ -84,9 +91,8 @@ async function run() { ); } - catch (e: any) { + catch (e) { error(`Unhandled task exception: ${e}`); - console.log(e); setResult(TaskResult.Failed, e?.message); } finally { diff --git a/extension/tasks/utils/dependabotUpdater.ts b/extension/tasks/utils/dependabotUpdater.ts index 1b37a736..b902f836 100644 --- a/extension/tasks/utils/dependabotUpdater.ts +++ b/extension/tasks/utils/dependabotUpdater.ts @@ -7,30 +7,100 @@ import * as fs from 'fs'; import * as os from 'os'; export interface IUpdateJobConfig { - id: string, job: { + // See: https://github.com/dependabot/dependabot-core/blob/main/updater/lib/dependabot/job.rb + 'id': string, 'package-manager': string, - 'allowed-updates': { - 'update-type': string + 'updating-a-pull-request': boolean, + 'dependency-group-to-refresh'?: string, + 'dependency-groups'?: { + 'name': string, + 'applies-to'?: string, + 'update-types'?: string[], + 'rules': { + 'patterns'?: string[] + 'exclude-patterns'?: string[], + 'dependency-type'?: string + }[] }[], - source: { - provider: string, - repo: string, - directory: string, - commit: string - } + 'dependencies'?: string[], + 'allowed-updates'?: { + 'dependency-name'?: string, + 'dependency-type'?: string, + 'update-type'?: string + }[], + 'ignore-conditions'?: { + 'dependency-name'?: string, + 'version-requirement'?: string, + 'source'?: string, + 'update-types'?: string[] + }[], + 'security-updates-only': boolean, + 'security-advisories'?: { + 'dependency-name': string, + 'affected-versions': string[], + 'patched-versions': string[], + 'unaffected-versions': string[], + 'title'?: string, + 'description'?: string, + 'source-name'?: string, + 'source-url'?: string + }[], + 'source': { + 'provider': string, + 'api-endpoint'?: string, + 'hostname': string, + 'repo': string, + 'branch'?: string, + 'commit'?: string, + 'directory'?: string, + 'directories'?: string[] + }, + 'existing-pull-requests'?: { + 'dependencies': { + 'name': string, + 'version'?: string, + 'removed': boolean, + 'directory'?: string + }[] + }, + 'existing-group-pull-requests'?: { + 'dependency-group-name': string, + 'dependencies': { + 'name': string, + 'version'?: string, + 'removed': boolean, + 'directory'?: string + }[] + }, + 'commit-message-options'?: { + 'prefix'?: string, + 'prefix-development'?: string, + 'include'?: string, + }, + 'experiments'?: any, + 'max-updater-run-time'?: number, + 'reject-external-code'?: boolean, + 'repo-contents-path'?: string, + 'requirements-update-strategy'?: string, + 'lockfile-only'?: boolean }, credentials: { - type: string, - host?: string, - username?: string, - password?: string, - url?: string, - token?: string + // See: https://github.com/dependabot/dependabot-core/blob/main/common/lib/dependabot/credential.rb + 'type': string, + 'host'?: string, + 'region'?: string, + 'url'?: string, + 'registry'?: string, + 'username'?: string, + 'password'?: string, + 'token'?: string, + 'replaces-base'?: boolean }[] } export interface IUpdateScenarioOutput { + // See: https://github.com/dependabot/smoke-tests/tree/main/tests type: string, data: any } @@ -48,18 +118,20 @@ export class DependabotUpdater { } // Run dependabot update - public async update(options: { - job: IUpdateJobConfig, - collectorImage?: string, - proxyImage?: string, - updaterImage?: string - }): Promise { + public async update( + config: IUpdateJobConfig, + options?: { + collectorImage?: string, + proxyImage?: string, + updaterImage?: string + } + ): Promise { // Install dependabot if not already installed await this.ensureToolsAreInstalled(); // Create the job directory - const jobId = options.job.id; + const jobId = config.job.id; const jobPath = path.join(this.jobsPath, jobId.toString()); const jobInputPath = path.join(jobPath, 'job.yaml'); const jobOutputPath = path.join(jobPath, 'scenario.yaml'); @@ -69,19 +141,21 @@ export class DependabotUpdater { } // Generate the job input file - writeJobInput(jobInputPath, options.job); + writeJobInput(jobInputPath, config); // Compile dependabot cmd arguments + // See: https://github.com/dependabot/cli/blob/main/cmd/dependabot/internal/cmd/root.go + // https://github.com/dependabot/cli/blob/main/cmd/dependabot/internal/cmd/update.go let dependabotArguments = [ "update", "-f", jobInputPath, "-o", jobOutputPath ]; - if (options.collectorImage) { + if (options?.collectorImage) { dependabotArguments.push("--collector-image", options.collectorImage); } - if (options.proxyImage) { + if (options?.proxyImage) { dependabotArguments.push("--proxy-image", options.proxyImage); } - if (options.updaterImage) { + if (options?.updaterImage) { dependabotArguments.push("--updater-image", options.updaterImage); } @@ -131,18 +205,18 @@ export class DependabotUpdater { } } -function writeJobInput(path: string, job: IUpdateJobConfig): void { - fs.writeFileSync(path, yaml.dump(job)); +function writeJobInput(path: string, config: IUpdateJobConfig): void { + fs.writeFileSync(path, yaml.dump(config)); } -function readScenarioOutputs(scenarioFilePath: string): IUpdateScenarioOutput[] { - if (!scenarioFilePath) { +function readScenarioOutputs(path: string): IUpdateScenarioOutput[] { + if (!path) { throw new Error("Scenario file path is required"); } - const scenarioContent = fs.readFileSync(scenarioFilePath, 'utf-8'); + const scenarioContent = fs.readFileSync(path, 'utf-8'); if (!scenarioContent || typeof scenarioContent !== 'string') { - throw new Error(`Scenario file could not be read at '${scenarioFilePath}'`); + throw new Error(`Scenario file could not be read at '${path}'`); } const scenario: any = yaml.load(scenarioContent); diff --git a/extension/tasks/utils/getSharedVariables.ts b/extension/tasks/utils/getSharedVariables.ts index 87a90ab6..34100a26 100644 --- a/extension/tasks/utils/getSharedVariables.ts +++ b/extension/tasks/utils/getSharedVariables.ts @@ -27,6 +27,9 @@ export interface ISharedVariables { /** Whether the repository was overridden via input */ repositoryOverridden: boolean; + /** Organisation API endpoint URL */ + apiEndpointUrl: string; + /** The github token */ githubAccessToken: string; /** The access User for Azure DevOps Repos */ @@ -86,9 +89,9 @@ export interface ISharedVariables { */ export default function getSharedVariables(): ISharedVariables { let organizationUrl = tl.getVariable('System.TeamFoundationCollectionUri'); + //convert url string into a valid JS URL object let formattedOrganizationUrl = new URL(organizationUrl); - let protocol: string = formattedOrganizationUrl.protocol.slice(0, -1); let hostname: string = extractHostname(formattedOrganizationUrl); let port: string = formattedOrganizationUrl.port; @@ -103,6 +106,9 @@ export default function getSharedVariables(): ISharedVariables { } repository = encodeURI(repository); // encode special characters like spaces + const virtualDirectorySuffix = virtualDirectory?.length > 0 ? `${virtualDirectory}/` : ''; + let apiEndpointUrl = `${protocol}://${hostname}:${port}/${virtualDirectorySuffix}`; + // Prepare the access credentials let githubAccessToken: string = getGithubAccessToken(); let systemAccessUser: string = tl.getInput('azureDevOpsUser'); @@ -153,6 +159,8 @@ export default function getSharedVariables(): ISharedVariables { repository, repositoryOverridden, + apiEndpointUrl, + githubAccessToken, systemAccessUser, systemAccessToken, From 943c35c9fb2aa49981c9d3b6bc5a5653039f66d2 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Mon, 2 Sep 2024 16:52:27 +1200 Subject: [PATCH 04/57] Move update output processing to dedicated class; Add DevOps API client --- extension/package-lock.json | 233 ++++++++++++++++++ extension/package.json | 1 + .../tasks/dependabot/dependabotV2/index.ts | 127 +++------- extension/tasks/utils/azureDevOpsApiClient.ts | 19 ++ .../azureDevOpsDependabotOutputProcessor.ts | 73 ++++++ extension/tasks/utils/dependabotTypes.ts | 107 ++++++++ extension/tasks/utils/dependabotUpdater.ts | 170 ++++--------- 7 files changed, 516 insertions(+), 214 deletions(-) create mode 100644 extension/tasks/utils/azureDevOpsApiClient.ts create mode 100644 extension/tasks/utils/azureDevOpsDependabotOutputProcessor.ts create mode 100644 extension/tasks/utils/dependabotTypes.ts diff --git a/extension/package-lock.json b/extension/package-lock.json index cf23f5b9..ed4141b6 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -10,6 +10,7 @@ "license": "MIT", "dependencies": { "axios": "1.7.5", + "azure-devops-node-api": "^14.0.2", "azure-pipelines-task-lib": "4.17.0", "js-yaml": "4.1.0" }, @@ -1311,6 +1312,18 @@ "proxy-from-env": "^1.1.0" } }, + "node_modules/azure-devops-node-api": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/azure-devops-node-api/-/azure-devops-node-api-14.0.2.tgz", + "integrity": "sha512-TwjAEnWnOSZ2oypkDyqppgvJw43qArEfPiJtEWLL3NBgdvAuOuB0xgFz/Eiz4H6Dk0Yv52wCodZxtZvAMhJXwQ==", + "dependencies": { + "tunnel": "0.0.6", + "typed-rest-client": "^2.0.1" + }, + "engines": { + "node": ">= 16.0.0" + } + }, "node_modules/azure-pipelines-task-lib": { "version": "4.17.0", "resolved": "https://registry.npmjs.org/azure-pipelines-task-lib/-/azure-pipelines-task-lib-4.17.0.tgz", @@ -1527,6 +1540,24 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "node_modules/call-bind": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", + "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -1761,6 +1792,22 @@ "node": ">=0.10.0" } }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -1769,6 +1816,15 @@ "node": ">=0.4.0" } }, + "node_modules/des.js": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.1.0.tgz", + "integrity": "sha512-r17GxjhUCjSRy8aiJpr8/UadFIzMzJGexI3Nmz4ADi9LYSFx4gTBp80+NaX/YsXWWLhpZ7v/v/ubEc/bCNfKwg==", + "dependencies": { + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -1845,6 +1901,25 @@ "is-arrayish": "^0.2.1" } }, + "node_modules/es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "dependencies": { + "get-intrinsic": "^1.2.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/escalade": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", @@ -2074,6 +2149,24 @@ "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/get-intrinsic": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/get-package-type": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", @@ -2135,6 +2228,17 @@ "node": ">=4" } }, + "node_modules/gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "dependencies": { + "get-intrinsic": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -2150,6 +2254,39 @@ "node": ">=8" } }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-proto": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", + "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -2978,6 +3115,11 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/js-md4": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/js-md4/-/js-md4-0.3.2.tgz", + "integrity": "sha512-/GDnfQYsltsjRswQhN9fhv3EMw2sCpUdrdxyWDOUK7eyD++r3gRhzgiQgc/x4MAv2i1iuQ4lxO5mvqM3vj4bwA==" + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -3165,6 +3307,11 @@ "node": ">=6" } }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + }, "node_modules/minimatch": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.5.tgz", @@ -3231,6 +3378,17 @@ "node": ">=8" } }, + "node_modules/object-inspect": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", + "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -3463,6 +3621,20 @@ "teleport": ">=0.2.0" } }, + "node_modules/qs": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "dependencies": { + "side-channel": "^1.0.6" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/react-is": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", @@ -3551,6 +3723,22 @@ "semver": "bin/semver" } }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -3588,6 +3776,23 @@ "node": ">=4" } }, + "node_modules/side-channel": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "dependencies": { + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -3892,6 +4097,14 @@ } } }, + "node_modules/tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", + "engines": { + "node": ">=0.6.11 <=0.7.0 || >=0.7.3" + } + }, "node_modules/type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", @@ -3913,6 +4126,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/typed-rest-client": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/typed-rest-client/-/typed-rest-client-2.0.2.tgz", + "integrity": "sha512-rmAQM2gZw/PQpK5+5aSs+I6ZBv4PFC2BT1o+0ADS1SgSejA+14EmbI2Lt8uXwkX7oeOMkwFmg0pHKwe8D9IT5A==", + "dependencies": { + "des.js": "^1.1.0", + "js-md4": "^0.3.2", + "qs": "^6.10.3", + "tunnel": "0.0.6", + "underscore": "^1.12.1" + }, + "engines": { + "node": ">= 16.0.0" + } + }, "node_modules/typescript": { "version": "5.5.4", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", @@ -3927,6 +4155,11 @@ "node": ">=14.17" } }, + "node_modules/underscore": { + "version": "1.13.7", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.7.tgz", + "integrity": "sha512-GMXzWtsc57XAtguZgaQViUOzs0KTkk8ojr3/xAxXLITqf/3EMwxC0inyETfDFjH/Krbhuep0HNbbjI9i/q3F3g==" + }, "node_modules/undici-types": { "version": "6.19.6", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.6.tgz", diff --git a/extension/package.json b/extension/package.json index 577fe195..9854b565 100644 --- a/extension/package.json +++ b/extension/package.json @@ -28,6 +28,7 @@ "dependencies": { "axios": "1.7.5", "azure-pipelines-task-lib": "4.17.0", + "azure-devops-node-api": "^14.0.2", "js-yaml": "4.1.0" }, "devDependencies": { diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index 51347be9..f6650363 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -1,8 +1,11 @@ import { which, setResult, TaskResult } from "azure-pipelines-task-lib/task" import { debug, warning, error } from "azure-pipelines-task-lib/task" -import { DependabotUpdater, IUpdateJobConfig, IUpdateScenarioOutput } from '../../utils/dependabotUpdater'; +import { IDependabotUpdateJob } from "../../utils/dependabotTypes"; +import { DependabotUpdater } from '../../utils/dependabotUpdater'; +import { AzureDevOpsClient } from "../../utils/azureDevOpsApiClient"; +import { AzureDevOpsDependabotOutputProcessor } from "../../utils/azureDevOpsDependabotOutputProcessor"; import { parseConfigFile } from '../../utils/parseConfigFile'; -import getSharedVariables from '../../utils/getSharedVariables'; +import getSharedVariables from '../../utils/getSharedVariables'; async function run() { let updater: DependabotUpdater = undefined; @@ -19,32 +22,37 @@ async function run() { const config = await parseConfigFile(variables); // Initialise the dependabot updater - // TODO: Add config for CLI image argument - updater = new DependabotUpdater(null, variables.debug); + updater = new DependabotUpdater( + DependabotUpdater.CLI_IMAGE_LATEST, // TODO: Add config for this? + new AzureDevOpsDependabotOutputProcessor( + new AzureDevOpsClient(variables.apiEndpointUrl, variables.systemAccessToken) + ), + variables.debug + ); // Process the updates per-ecosystem - let updatedSuccessfully: boolean = true; + let taskWasSuccessful: boolean = true; config.updates.forEach(async (update) => { // TODO: Fetch all existing PRs from DevOps - + let registryCredentials = new Array(); for (const key in config.registries) { const registry = config.registries[key]; registryCredentials.push({ - type: registry.type, - host: registry.host, - region: undefined, // TODO: registry.region, - url: registry.url, - registry: registry.registry, - username: registry.username, - password: registry.password, - token: registry.token, - 'replaces-base': registry['replaces-base'] + type: registry.type, + host: registry.host, + region: undefined, // TODO: registry.region, + url: registry.url, + registry: registry.registry, + username: registry.username, + password: registry.password, + token: registry.token, + 'replaces-base': registry['replaces-base'] }); }; - let job: IUpdateJobConfig = { + let job: IDependabotUpdateJob = { job: { // TODO: Parse all options from `config` and `variables` id: 'job-1', @@ -55,14 +63,14 @@ async function run() { ], 'security-updates-only': false, source: { - provider: 'azure', - 'api-endpoint': variables.apiEndpointUrl, - hostname: variables.hostname, - repo: `${variables.organization}/${variables.project}/_git/${variables.repository}`, - branch: update.targetBranch, // TODO: add config for 'source branch'?? - commit: undefined, // TODO: add config for this? - directory: update.directories?.length == 0 ? update.directory : undefined, - directories: update.directories?.length > 0 ? update.directories : undefined + provider: 'azure', + 'api-endpoint': variables.apiEndpointUrl, + hostname: variables.hostname, + repo: `${variables.organization}/${variables.project}/_git/${variables.repository}`, + branch: update.targetBranch, // TODO: add config for 'source branch'?? + commit: undefined, // TODO: add config for this? + directory: update.directories?.length == 0 ? update.directory : undefined, + directories: update.directories?.length > 0 ? update.directories : undefined } }, credentials: (registryCredentials || []).concat([ @@ -76,8 +84,8 @@ async function run() { }; // Run dependabot updater for the job - if (!processUpdateOutputs(await updater.update(job))) { - updatedSuccessfully = false; + if ((await updater.update(job)).filter(u => !u.success).length > 0) { + taskWasSuccessful = false; } // TODO: Loop through all existing PRs and do a single update job for each, update/close the PR as needed @@ -86,8 +94,8 @@ async function run() { }); setResult( - updatedSuccessfully ? TaskResult.Succeeded : TaskResult.Failed, - updatedSuccessfully ? 'All update jobs completed successfully' : 'One or more update jobs failed, check logs for more information' + taskWasSuccessful ? TaskResult.Succeeded : TaskResult.Failed, + taskWasSuccessful ? 'All update jobs completed successfully' : 'One or more update jobs failed, check logs for more information' ); } @@ -100,67 +108,4 @@ async function run() { } } -// Process the job outputs and apply changes to DevOps -// TODO: Move this to a new util class, e.g. `dependabotOutputProcessor.ts` -function processUpdateOutputs(outputs: IUpdateScenarioOutput[]) : boolean { - let success: boolean = true; - outputs.forEach(output => { - switch (output.type) { - - case 'update_dependency_list': - console.log('TODO: UPDATED DEPENDENCY LIST: ', output.data); - // TODO: Save data to DevOps? This would be really useful for generating a dependency graph hub page or HTML report (future feature maybe?) - break; - - case 'create_pull_request': - console.log('TODO: CREATE PULL REQUEST: ', output.data); - // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: create_pull_request() - break; - - case 'update_pull_request': - console.log('TODO: UPDATE PULL REQUEST ', output.data); - // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: update_pull_request() - break; - - case 'close_pull_request': - console.log('TODO: CLOSE PULL REQUEST ', output.data); - // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: close_pull_request() - break; - - case 'mark_as_processed': - console.log('TODO: MARK AS PROCESSED: ', output.data); - // TODO: Log info? - break; - - case 'record_ecosystem_versions': - console.log('TODO: RECORD ECOSYSTEM VERSIONS: ', output.data); - // TODO: Log info? - break; - - case 'record_update_job_error': - console.log('TODO: RECORD UPDATE JOB ERROR: ', output.data); - // TODO: Log error? - success = false; - break; - - case 'record_update_job_unknown_error': - console.log('TODO: RECORD UPDATE JOB UNKNOWN ERROR: ', output.data); - // TODO: Log error? - success = false; - break; - - case 'increment_metric': - console.log('TODO: INCREMENT METRIC: ', output.data); - // TODO: Log info? - break; - - default: - warning(`Unknown dependabot output type: ${output.type}`); - success = false; - break; - } - }); - return success; -} - run(); diff --git a/extension/tasks/utils/azureDevOpsApiClient.ts b/extension/tasks/utils/azureDevOpsApiClient.ts new file mode 100644 index 00000000..96f97df8 --- /dev/null +++ b/extension/tasks/utils/azureDevOpsApiClient.ts @@ -0,0 +1,19 @@ +import { debug, warning, error } from "azure-pipelines-task-lib/task" +import { WebApi, getPersonalAccessTokenHandler } from "azure-devops-node-api"; + +// Wrapper for DevOps API actions +export class AzureDevOpsClient { + private readonly connection: WebApi; + private userId: string | null = null; + + constructor(apiEndpointUrl: string, accessToken: string) { + this.connection = new WebApi( + apiEndpointUrl, + getPersonalAccessTokenHandler(accessToken) + ); + } + + private async getUserId(): Promise { + return (this.userId ||= (await this.connection.connect()).authenticatedUser?.id || ""); + } +} \ No newline at end of file diff --git a/extension/tasks/utils/azureDevOpsDependabotOutputProcessor.ts b/extension/tasks/utils/azureDevOpsDependabotOutputProcessor.ts new file mode 100644 index 00000000..0471ecb0 --- /dev/null +++ b/extension/tasks/utils/azureDevOpsDependabotOutputProcessor.ts @@ -0,0 +1,73 @@ +import { debug, warning, error } from "azure-pipelines-task-lib/task" +import { IDependabotUpdateOutputProcessor } from "./dependabotTypes"; +import { AzureDevOpsClient } from "./azureDevOpsApiClient"; + +// Processes dependabot update outputs using the DevOps API +export class AzureDevOpsDependabotOutputProcessor implements IDependabotUpdateOutputProcessor { + private readonly api: AzureDevOpsClient; + + constructor(api: AzureDevOpsClient) { + this.api = api; + } + + // Process the appropriate DevOps API actions for the supplied dependabot update output + public async process(type: string, data: any): Promise { + let success: boolean = true; + switch (type) { + + case 'update_dependency_list': + console.log('TODO: UPDATED DEPENDENCY LIST: ', data); + // TODO: Save data to DevOps? This would be really useful for generating a dependency graph hub page or HTML report (future feature maybe?) + break; + + case 'create_pull_request': + console.log('TODO: CREATE PULL REQUEST: ', data); + // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: create_pull_request() + break; + + case 'update_pull_request': + console.log('TODO: UPDATE PULL REQUEST ', data); + // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: update_pull_request() + break; + + case 'close_pull_request': + console.log('TODO: CLOSE PULL REQUEST ', data); + // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: close_pull_request() + break; + + case 'mark_as_processed': + console.log('TODO: MARK AS PROCESSED: ', data); + // TODO: Log info? + break; + + case 'record_ecosystem_versions': + console.log('TODO: RECORD ECOSYSTEM VERSIONS: ', data); + // TODO: Log info? + break; + + case 'record_update_job_error': + console.log('TODO: RECORD UPDATE JOB ERROR: ', data); + // TODO: Log error? + success = false; + break; + + case 'record_update_job_unknown_error': + console.log('TODO: RECORD UPDATE JOB UNKNOWN ERROR: ', data); + // TODO: Log error? + success = false; + break; + + case 'increment_metric': + console.log('TODO: INCREMENT METRIC: ', data); + // TODO: Log info? + break; + + default: + warning(`Unknown dependabot output type: ${type}`); + success = false; + break; + } + + return success; + } +} \ No newline at end of file diff --git a/extension/tasks/utils/dependabotTypes.ts b/extension/tasks/utils/dependabotTypes.ts new file mode 100644 index 00000000..c9c8eccc --- /dev/null +++ b/extension/tasks/utils/dependabotTypes.ts @@ -0,0 +1,107 @@ + +export interface IDependabotUpdateJob { + job: { + // See: https://github.com/dependabot/dependabot-core/blob/main/updater/lib/dependabot/job.rb + 'id': string, + 'package-manager': string, + 'updating-a-pull-request': boolean, + 'dependency-group-to-refresh'?: string, + 'dependency-groups'?: { + 'name': string, + 'applies-to'?: string, + 'update-types'?: string[], + 'rules': { + 'patterns'?: string[] + 'exclude-patterns'?: string[], + 'dependency-type'?: string + }[] + }[], + 'dependencies'?: string[], + 'allowed-updates'?: { + 'dependency-name'?: string, + 'dependency-type'?: string, + 'update-type'?: string + }[], + 'ignore-conditions'?: { + 'dependency-name'?: string, + 'version-requirement'?: string, + 'source'?: string, + 'update-types'?: string[] + }[], + 'security-updates-only': boolean, + 'security-advisories'?: { + 'dependency-name': string, + 'affected-versions': string[], + 'patched-versions': string[], + 'unaffected-versions': string[], + 'title'?: string, + 'description'?: string, + 'source-name'?: string, + 'source-url'?: string + }[], + 'source': { + 'provider': string, + 'api-endpoint'?: string, + 'hostname': string, + 'repo': string, + 'branch'?: string, + 'commit'?: string, + 'directory'?: string, + 'directories'?: string[] + }, + 'existing-pull-requests'?: { + 'dependencies': { + 'name': string, + 'version'?: string, + 'removed': boolean, + 'directory'?: string + }[] + }, + 'existing-group-pull-requests'?: { + 'dependency-group-name': string, + 'dependencies': { + 'name': string, + 'version'?: string, + 'removed': boolean, + 'directory'?: string + }[] + }, + 'commit-message-options'?: { + 'prefix'?: string, + 'prefix-development'?: string, + 'include'?: string, + }, + 'experiments'?: any, + 'max-updater-run-time'?: number, + 'reject-external-code'?: boolean, + 'repo-contents-path'?: string, + 'requirements-update-strategy'?: string, + 'lockfile-only'?: boolean + }, + credentials: { + // See: https://github.com/dependabot/dependabot-core/blob/main/common/lib/dependabot/credential.rb + 'type': string, + 'host'?: string, + 'region'?: string, + 'url'?: string, + 'registry'?: string, + 'username'?: string, + 'password'?: string, + 'token'?: string, + 'replaces-base'?: boolean + }[] +} + +export interface IDependabotUpdateOutput { + success: boolean, + error: any, + output: { + // See: https://github.com/dependabot/smoke-tests/tree/main/tests + type: string, + data: any + } +} + +export interface IDependabotUpdateOutputProcessor { + process(type: string, data: any): Promise; +} diff --git a/extension/tasks/utils/dependabotUpdater.ts b/extension/tasks/utils/dependabotUpdater.ts index b902f836..94376765 100644 --- a/extension/tasks/utils/dependabotUpdater.ts +++ b/extension/tasks/utils/dependabotUpdater.ts @@ -1,131 +1,38 @@ import { debug, warning, error } from "azure-pipelines-task-lib/task" import { which, tool } from "azure-pipelines-task-lib/task" import { ToolRunner } from "azure-pipelines-task-lib/toolrunner" +import { IDependabotUpdateOutputProcessor, IDependabotUpdateJob, IDependabotUpdateOutput } from "./dependabotTypes"; import * as yaml from 'js-yaml'; import * as path from 'path'; import * as fs from 'fs'; import * as os from 'os'; -export interface IUpdateJobConfig { - job: { - // See: https://github.com/dependabot/dependabot-core/blob/main/updater/lib/dependabot/job.rb - 'id': string, - 'package-manager': string, - 'updating-a-pull-request': boolean, - 'dependency-group-to-refresh'?: string, - 'dependency-groups'?: { - 'name': string, - 'applies-to'?: string, - 'update-types'?: string[], - 'rules': { - 'patterns'?: string[] - 'exclude-patterns'?: string[], - 'dependency-type'?: string - }[] - }[], - 'dependencies'?: string[], - 'allowed-updates'?: { - 'dependency-name'?: string, - 'dependency-type'?: string, - 'update-type'?: string - }[], - 'ignore-conditions'?: { - 'dependency-name'?: string, - 'version-requirement'?: string, - 'source'?: string, - 'update-types'?: string[] - }[], - 'security-updates-only': boolean, - 'security-advisories'?: { - 'dependency-name': string, - 'affected-versions': string[], - 'patched-versions': string[], - 'unaffected-versions': string[], - 'title'?: string, - 'description'?: string, - 'source-name'?: string, - 'source-url'?: string - }[], - 'source': { - 'provider': string, - 'api-endpoint'?: string, - 'hostname': string, - 'repo': string, - 'branch'?: string, - 'commit'?: string, - 'directory'?: string, - 'directories'?: string[] - }, - 'existing-pull-requests'?: { - 'dependencies': { - 'name': string, - 'version'?: string, - 'removed': boolean, - 'directory'?: string - }[] - }, - 'existing-group-pull-requests'?: { - 'dependency-group-name': string, - 'dependencies': { - 'name': string, - 'version'?: string, - 'removed': boolean, - 'directory'?: string - }[] - }, - 'commit-message-options'?: { - 'prefix'?: string, - 'prefix-development'?: string, - 'include'?: string, - }, - 'experiments'?: any, - 'max-updater-run-time'?: number, - 'reject-external-code'?: boolean, - 'repo-contents-path'?: string, - 'requirements-update-strategy'?: string, - 'lockfile-only'?: boolean - }, - credentials: { - // See: https://github.com/dependabot/dependabot-core/blob/main/common/lib/dependabot/credential.rb - 'type': string, - 'host'?: string, - 'region'?: string, - 'url'?: string, - 'registry'?: string, - 'username'?: string, - 'password'?: string, - 'token'?: string, - 'replaces-base'?: boolean - }[] -} - -export interface IUpdateScenarioOutput { - // See: https://github.com/dependabot/smoke-tests/tree/main/tests - type: string, - data: any -} - +// Wrapper class for running updates using dependabot-cli export class DependabotUpdater { private readonly jobsPath: string; private readonly toolImage: string; + private readonly outputProcessor: IDependabotUpdateOutputProcessor; private readonly debug: boolean; - constructor(cliToolImage?: string, debug?: boolean) { + public static readonly CLI_IMAGE_LATEST = "github.com/dependabot/cli/cmd/dependabot@latest"; + + constructor(cliToolImage: string, outputProcessor: IDependabotUpdateOutputProcessor, debug: boolean) { this.jobsPath = path.join(os.tmpdir(), 'dependabot-jobs'); - this.toolImage = cliToolImage ?? "github.com/dependabot/cli/cmd/dependabot@latest"; - this.debug = debug ?? false; + this.toolImage = cliToolImage; + this.outputProcessor = outputProcessor; + this.debug = debug; this.ensureJobsPathExists(); } // Run dependabot update public async update( - config: IUpdateJobConfig, + config: IDependabotUpdateJob, options?: { collectorImage?: string, proxyImage?: string, updaterImage?: string } - ): Promise { + ): Promise { // Install dependabot if not already installed await this.ensureToolsAreInstalled(); @@ -136,8 +43,8 @@ export class DependabotUpdater { const jobInputPath = path.join(jobPath, 'job.yaml'); const jobOutputPath = path.join(jobPath, 'scenario.yaml'); this.ensureJobsPathExists(); - if (!fs.existsSync(jobPath)){ - fs.mkdirSync(jobPath); + if (!fs.existsSync(jobPath)) { + fs.mkdirSync(jobPath); } // Generate the job input file @@ -168,7 +75,32 @@ export class DependabotUpdater { throw new Error(`Dependabot update failed with exit code ${dependabotResultCode}`); } - return readScenarioOutputs(jobOutputPath); + console.info("Processing dependabot update outputs..."); + const processedOutputs = Array(); + for (const output of readScenarioOutputs(jobOutputPath)) { + const type = output['type']; + const data = output['expect']?.['data']; + var processedOutput = { + success: true, + error: null, + output: { + type: type, + data: data + } + }; + try { + processedOutput.success = await this.outputProcessor.process(type, data); + } + catch (e) { + processedOutput.success = false; + processedOutput.error = e; + } + finally { + processedOutputs.push(processedOutput); + } + } + + return processedOutputs; } // Install dependabot if not already installed @@ -189,14 +121,14 @@ export class DependabotUpdater { // Create the jobs directory if it does not exist private ensureJobsPathExists(): void { - if (!fs.existsSync(this.jobsPath)){ + if (!fs.existsSync(this.jobsPath)) { fs.mkdirSync(this.jobsPath); } } // Clean up the jobs directory and its contents public cleanup(): void { - if (fs.existsSync(this.jobsPath)){ + if (fs.existsSync(this.jobsPath)) { fs.rmSync(this.jobsPath, { recursive: true, force: true @@ -205,32 +137,24 @@ export class DependabotUpdater { } } -function writeJobInput(path: string, config: IUpdateJobConfig): void { +function writeJobInput(path: string, config: IDependabotUpdateJob): void { fs.writeFileSync(path, yaml.dump(config)); } -function readScenarioOutputs(path: string): IUpdateScenarioOutput[] { +function readScenarioOutputs(path: string): any[] { if (!path) { throw new Error("Scenario file path is required"); } const scenarioContent = fs.readFileSync(path, 'utf-8'); if (!scenarioContent || typeof scenarioContent !== 'string') { - throw new Error(`Scenario file could not be read at '${path}'`); + throw new Error(`Scenario file could not be read at '${path}'`); } - + const scenario: any = yaml.load(scenarioContent); if (scenario === null || typeof scenario !== 'object') { - throw new Error('Invalid scenario object'); + throw new Error('Invalid scenario object'); } - - let outputs = new Array(); - scenario['output']?.forEach((output: any) => { - outputs.push({ - type: output['type'], - data: output['expect']?.['data'] - }); - }); - return outputs; + return scenario['output'] || []; } \ No newline at end of file From 7de7611f96eedf64aae84d05e4a5fb803e64b3ef Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Mon, 2 Sep 2024 16:54:11 +1200 Subject: [PATCH 05/57] Codespell skip package-lock.json --- .codespellrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.codespellrc b/.codespellrc index 13c05a80..21ec3a65 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,5 +1,5 @@ [codespell] -skip = .git,*.pdf,*.svg,pnpm-lock.yaml,yarn.lock +skip = .git,*.pdf,*.svg,pnpm-lock.yaml,yarn.lock,package-lock.json # some modules, parts of regexes, and variable names to ignore, some # misspellings in fixtures/external responses we do not own ignore-words-list = caf,bu,nwo,nd,kernal,crate,unparseable,couldn,defintions From cdbe00434d2ac9a983ee832d0eee0993d6a7f4ff Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Mon, 2 Sep 2024 17:06:19 +1200 Subject: [PATCH 06/57] Clean-up --- .../tasks/dependabot/dependabotV2/index.ts | 29 ++++++++++--------- extension/tasks/utils/azureDevOpsApiClient.ts | 4 +-- extension/tasks/utils/dependabotTypes.ts | 2 +- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index f6650363..b94e7cec 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -8,7 +8,7 @@ import { parseConfigFile } from '../../utils/parseConfigFile'; import getSharedVariables from '../../utils/getSharedVariables'; async function run() { - let updater: DependabotUpdater = undefined; + let dependabot: DependabotUpdater = undefined; try { // Check if required tools are installed @@ -17,16 +17,19 @@ async function run() { debug('Checking for `go` install...'); which('go', true); - // Parse the dependabot configuration file + // Parse the dependabot.yaml configuration file const variables = getSharedVariables(); const config = await parseConfigFile(variables); - // Initialise the dependabot updater - updater = new DependabotUpdater( + // Initialise the DevOps API client + const api = new AzureDevOpsClient( + variables.organizationUrl.toString(), variables.systemAccessToken + ); + + // Initialise the Dependabot updater + dependabot = new DependabotUpdater( DependabotUpdater.CLI_IMAGE_LATEST, // TODO: Add config for this? - new AzureDevOpsDependabotOutputProcessor( - new AzureDevOpsClient(variables.apiEndpointUrl, variables.systemAccessToken) - ), + new AzureDevOpsDependabotOutputProcessor(api), variables.debug ); @@ -42,24 +45,24 @@ async function run() { registryCredentials.push({ type: registry.type, host: registry.host, - region: undefined, // TODO: registry.region, url: registry.url, registry: registry.registry, + region: undefined, // TODO: registry.region, username: registry.username, password: registry.password, token: registry.token, - 'replaces-base': registry['replaces-base'] + 'replaces-base': registry['replaces-base'] || false }); }; let job: IDependabotUpdateJob = { job: { // TODO: Parse all options from `config` and `variables` - id: 'job-1', + id: 'job-1', // TODO: Make timestamp or auto-incrementing id? 'package-manager': update.packageEcosystem, 'updating-a-pull-request': false, 'allowed-updates': [ - { 'update-type': 'all' } + { 'update-type': 'all' } // TODO: update.allow ], 'security-updates-only': false, source: { @@ -84,7 +87,7 @@ async function run() { }; // Run dependabot updater for the job - if ((await updater.update(job)).filter(u => !u.success).length > 0) { + if ((await dependabot.update(job)).filter(u => !u.success).length > 0) { taskWasSuccessful = false; } @@ -104,7 +107,7 @@ async function run() { setResult(TaskResult.Failed, e?.message); } finally { - updater?.cleanup(); + dependabot?.cleanup(); } } diff --git a/extension/tasks/utils/azureDevOpsApiClient.ts b/extension/tasks/utils/azureDevOpsApiClient.ts index 96f97df8..3a587905 100644 --- a/extension/tasks/utils/azureDevOpsApiClient.ts +++ b/extension/tasks/utils/azureDevOpsApiClient.ts @@ -6,9 +6,9 @@ export class AzureDevOpsClient { private readonly connection: WebApi; private userId: string | null = null; - constructor(apiEndpointUrl: string, accessToken: string) { + constructor(apiUrl: string, accessToken: string) { this.connection = new WebApi( - apiEndpointUrl, + apiUrl, getPersonalAccessTokenHandler(accessToken) ); } diff --git a/extension/tasks/utils/dependabotTypes.ts b/extension/tasks/utils/dependabotTypes.ts index c9c8eccc..8dbe92c7 100644 --- a/extension/tasks/utils/dependabotTypes.ts +++ b/extension/tasks/utils/dependabotTypes.ts @@ -82,9 +82,9 @@ export interface IDependabotUpdateJob { // See: https://github.com/dependabot/dependabot-core/blob/main/common/lib/dependabot/credential.rb 'type': string, 'host'?: string, - 'region'?: string, 'url'?: string, 'registry'?: string, + 'region'?: string, 'username'?: string, 'password'?: string, 'token'?: string, From c49852d3123ac7e9940ec1a7a97a340452b9b29a Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Tue, 3 Sep 2024 13:55:35 +1200 Subject: [PATCH 07/57] Implement create pull request --- .../tasks/dependabot/dependabotV2/index.ts | 38 +++--- extension/tasks/utils/azureDevOpsApiClient.ts | 116 +++++++++++++++++- .../azureDevOpsDependabotOutputProcessor.ts | 102 +++++++++++---- extension/tasks/utils/dependabotTypes.ts | 2 +- extension/tasks/utils/dependabotUpdater.ts | 16 +-- 5 files changed, 225 insertions(+), 49 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index b94e7cec..f0fd0f28 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -8,7 +8,7 @@ import { parseConfigFile } from '../../utils/parseConfigFile'; import getSharedVariables from '../../utils/getSharedVariables'; async function run() { - let dependabot: DependabotUpdater = undefined; + let dependabotCli: DependabotUpdater = undefined; try { // Check if required tools are installed @@ -18,30 +18,30 @@ async function run() { which('go', true); // Parse the dependabot.yaml configuration file - const variables = getSharedVariables(); - const config = await parseConfigFile(variables); + const taskVariables = getSharedVariables(); + const dependabotConfig = await parseConfigFile(taskVariables); // Initialise the DevOps API client - const api = new AzureDevOpsClient( - variables.organizationUrl.toString(), variables.systemAccessToken + const azdoApi = new AzureDevOpsClient( + taskVariables.organizationUrl.toString(), taskVariables.systemAccessToken ); // Initialise the Dependabot updater - dependabot = new DependabotUpdater( + dependabotCli = new DependabotUpdater( DependabotUpdater.CLI_IMAGE_LATEST, // TODO: Add config for this? - new AzureDevOpsDependabotOutputProcessor(api), - variables.debug + new AzureDevOpsDependabotOutputProcessor(azdoApi, taskVariables), + taskVariables.debug ); // Process the updates per-ecosystem let taskWasSuccessful: boolean = true; - config.updates.forEach(async (update) => { + dependabotConfig.updates.forEach(async (update) => { // TODO: Fetch all existing PRs from DevOps let registryCredentials = new Array(); - for (const key in config.registries) { - const registry = config.registries[key]; + for (const key in dependabotConfig.registries) { + const registry = dependabotConfig.registries[key]; registryCredentials.push({ type: registry.type, host: registry.host, @@ -67,9 +67,9 @@ async function run() { 'security-updates-only': false, source: { provider: 'azure', - 'api-endpoint': variables.apiEndpointUrl, - hostname: variables.hostname, - repo: `${variables.organization}/${variables.project}/_git/${variables.repository}`, + 'api-endpoint': taskVariables.apiEndpointUrl, + hostname: taskVariables.hostname, + repo: `${taskVariables.organization}/${taskVariables.project}/_git/${taskVariables.repository}`, branch: update.targetBranch, // TODO: add config for 'source branch'?? commit: undefined, // TODO: add config for this? directory: update.directories?.length == 0 ? update.directory : undefined, @@ -79,15 +79,15 @@ async function run() { credentials: (registryCredentials || []).concat([ { type: 'git_source', - host: variables.hostname, - username: variables.systemAccessUser?.trim()?.length > 0 ? variables.systemAccessUser : 'x-access-token', - password: variables.systemAccessToken + host: taskVariables.hostname, + username: taskVariables.systemAccessUser?.trim()?.length > 0 ? taskVariables.systemAccessUser : 'x-access-token', + password: taskVariables.systemAccessToken } ]) }; // Run dependabot updater for the job - if ((await dependabot.update(job)).filter(u => !u.success).length > 0) { + if ((await dependabotCli.update(job)).filter(u => !u.success).length > 0) { taskWasSuccessful = false; } @@ -107,7 +107,7 @@ async function run() { setResult(TaskResult.Failed, e?.message); } finally { - dependabot?.cleanup(); + dependabotCli?.cleanup(); } } diff --git a/extension/tasks/utils/azureDevOpsApiClient.ts b/extension/tasks/utils/azureDevOpsApiClient.ts index 3a587905..df293955 100644 --- a/extension/tasks/utils/azureDevOpsApiClient.ts +++ b/extension/tasks/utils/azureDevOpsApiClient.ts @@ -1,8 +1,50 @@ import { debug, warning, error } from "azure-pipelines-task-lib/task" import { WebApi, getPersonalAccessTokenHandler } from "azure-devops-node-api"; +import { ItemContentType, VersionControlChangeType } from "azure-devops-node-api/interfaces/GitInterfaces"; + +export interface IPullRequest { + organisation: string, + project: string, + repository: string, + source: { + commit: string, + branch: string + }, + target: { + branch: string + }, + author: { + email: string, + name: string + }, + title: string, + description: string, + commitMessage: string, + mergeStrategy: string, + autoComplete: { + enabled: boolean, + bypassPolicyIds: number[], + }, + autoApprove: { + enabled: boolean, + approverUserToken: string + }, + assignees: string[], + reviewers: string[], + labels: string[], + workItems: number[], + dependencies: any, + changes: { + changeType: VersionControlChangeType, + path: string, + content: string, + encoding: string + }[] +}; // Wrapper for DevOps API actions export class AzureDevOpsClient { + private readonly connection: WebApi; private userId: string | null = null; @@ -13,7 +55,79 @@ export class AzureDevOpsClient { ); } + public async createPullRequest(pr: IPullRequest): Promise { + console.info(`Creating pull request for '${pr.title}'...`); + try { + const userId = await this.getUserId(); + const git = await this.connection.getGitApi(); + + // Create a new branch for the pull request and commit the changes + console.info(`Pushing ${pr.changes.length} change(s) to branch '${pr.source.branch}'...`); + const push = await git.createPush( + { + refUpdates: [ + { + name: `refs/heads/${pr.source.branch}`, + oldObjectId: pr.source.commit + } + ], + commits: [ + { + comment: pr.commitMessage, + author: { + email: pr.author.email, + name: pr.author.name + }, + changes: pr.changes.map(change => { + return { + changeType: change.changeType, + item: { + path: normalizeDevOpsPath(change.path) + }, + newContent: { + content: change.content, + contentType: ItemContentType.RawText + } + }; + }) + } + ] + }, + pr.repository, + pr.project + ); + + // Create the pull request + console.info(`Creating pull request to merge '${pr.source.branch}' into '${pr.target.branch}'...`); + const pullRequest = await git.createPullRequest( + { + sourceRefName: `refs/heads/${pr.target.branch}`, // Merge from the new dependabot update branch + targetRefName: `refs/heads/${pr.source.branch}`, // Merge to the original branch + title: pr.title, + description: pr.description + + }, + pr.repository, + pr.project, + true + ); + + console.info(`Pull request #${pullRequest?.pullRequestId} created successfully.`); + return pullRequest?.pullRequestId || null; + } + catch (e) { + error(`Failed to create pull request: ${e}`); + console.log(e); + return null; + } + } + private async getUserId(): Promise { return (this.userId ||= (await this.connection.connect()).authenticatedUser?.id || ""); } -} \ No newline at end of file +} + +function normalizeDevOpsPath(path: string): string { + // Convert backslashes to forward slashes, convert './' => '/' and ensure the path starts with a forward slash if it doesn't already, this is how DevOps paths are formatted + return path.replace(/\\/g, "/").replace(/^\.\//, "/").replace(/^([^/])/, "/$1"); +} diff --git a/extension/tasks/utils/azureDevOpsDependabotOutputProcessor.ts b/extension/tasks/utils/azureDevOpsDependabotOutputProcessor.ts index 0471ecb0..c9c5dabc 100644 --- a/extension/tasks/utils/azureDevOpsDependabotOutputProcessor.ts +++ b/extension/tasks/utils/azureDevOpsDependabotOutputProcessor.ts @@ -1,70 +1,130 @@ import { debug, warning, error } from "azure-pipelines-task-lib/task" -import { IDependabotUpdateOutputProcessor } from "./dependabotTypes"; +import { ISharedVariables } from "./getSharedVariables"; +import { IDependabotUpdateJob, IDependabotUpdateOutputProcessor } from "./dependabotTypes"; import { AzureDevOpsClient } from "./azureDevOpsApiClient"; +import { VersionControlChangeType } from "azure-devops-node-api/interfaces/GitInterfaces"; +import * as path from 'path'; // Processes dependabot update outputs using the DevOps API export class AzureDevOpsDependabotOutputProcessor implements IDependabotUpdateOutputProcessor { private readonly api: AzureDevOpsClient; + private readonly taskVariables: ISharedVariables; - constructor(api: AzureDevOpsClient) { + constructor(api: AzureDevOpsClient, taskVariables: ISharedVariables) { this.api = api; + this.taskVariables = taskVariables; } // Process the appropriate DevOps API actions for the supplied dependabot update output - public async process(type: string, data: any): Promise { + public async process(update: IDependabotUpdateJob, type: string, data: any): Promise { + console.debug(`Processing '${type}' with data:`, data); let success: boolean = true; switch (type) { case 'update_dependency_list': - console.log('TODO: UPDATED DEPENDENCY LIST: ', data); - // TODO: Save data to DevOps? This would be really useful for generating a dependency graph hub page or HTML report (future feature maybe?) + // TODO: Store dependency list info in DevOps project data? + // This could be used to generate a dependency graph hub/page/report (future feature maybe?) break; case 'create_pull_request': - console.log('TODO: CREATE PULL REQUEST: ', data); - // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: create_pull_request() + if (this.taskVariables.skipPullRequests) { + warning(`Skipping pull request creation as 'skipPullRequests' is set to 'true'`); + return; + } + // TODO: Skip if active pull request limit reached. + + const sourceRepoParts = update.job.source.repo.split('/'); // "{organisation}/{project}/_git/{repository}"" + await this.api.createPullRequest({ + organisation: sourceRepoParts[0], + project: sourceRepoParts[1], + repository: sourceRepoParts[3], + source: { + commit: data['base-commit-sha'] || update.job.source.commit, + branch: `dependabot/${update.job["package-manager"]}/${update.job.id}` // TODO: Get from dependabot.yaml + }, + target: { + branch: 'main' // TODO: Get from dependabot.yaml + }, + author: { + email: 'noreply@github.com', // TODO: Get from task variables? + name: 'dependabot[bot]', // TODO: Get from task variables? + }, + title: data['pr-title'], + description: data['pr-body'], + commitMessage: data['commit-message'], + mergeStrategy: this.taskVariables.mergeStrategy, + autoComplete: { + enabled: this.taskVariables.setAutoComplete, + bypassPolicyIds: this.taskVariables.autoCompleteIgnoreConfigIds + }, + autoApprove: { + enabled: this.taskVariables.autoApprove, + approverUserToken: this.taskVariables.autoApproveUserToken + }, + assignees: [], // TODO: Get from dependabot.yaml + reviewers: [], // TODO: Get from dependabot.yaml + labels: [], // TODO: Get from dependabot.yaml + workItems: [], // TODO: Get from dependabot.yaml + dependencies: data['dependencies'], + changes: data['updated-dependency-files'].filter((file) => file['type'] === 'file').map((file) => { + let changeType = VersionControlChangeType.None; + if (file['deleted'] === true) { + changeType = VersionControlChangeType.Delete; + } else if (file['operation'] === 'update') { + changeType = VersionControlChangeType.Edit; + } else { + changeType = VersionControlChangeType.Add; + } + return { + changeType: changeType, + path: path.join(file['directory'], file['name']), + content: file['content'], + encoding: file['content_encoding'] + } + }) + }) break; case 'update_pull_request': - console.log('TODO: UPDATE PULL REQUEST ', data); + if (this.taskVariables.skipPullRequests) { + warning(`Skipping pull request update as 'skipPullRequests' is set to 'true'`); + return; + } // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: update_pull_request() break; case 'close_pull_request': - console.log('TODO: CLOSE PULL REQUEST ', data); + if (this.taskVariables.abandonUnwantedPullRequests) { + warning(`Skipping pull request closure as 'abandonUnwantedPullRequests' is set to 'true'`); + return; + } // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: close_pull_request() break; case 'mark_as_processed': - console.log('TODO: MARK AS PROCESSED: ', data); - // TODO: Log info? + // TODO: Log this? break; case 'record_ecosystem_versions': - console.log('TODO: RECORD ECOSYSTEM VERSIONS: ', data); - // TODO: Log info? + // TODO: Log this? break; case 'record_update_job_error': - console.log('TODO: RECORD UPDATE JOB ERROR: ', data); - // TODO: Log error? + // TODO: Log this? success = false; break; case 'record_update_job_unknown_error': - console.log('TODO: RECORD UPDATE JOB UNKNOWN ERROR: ', data); - // TODO: Log error? + // TODO: Log this? success = false; break; case 'increment_metric': - console.log('TODO: INCREMENT METRIC: ', data); - // TODO: Log info? + // TODO: Log this? break; default: - warning(`Unknown dependabot output type: ${type}`); - success = false; + warning(`Unknown dependabot output type '${type}', ignoring...`); break; } diff --git a/extension/tasks/utils/dependabotTypes.ts b/extension/tasks/utils/dependabotTypes.ts index 8dbe92c7..31eba762 100644 --- a/extension/tasks/utils/dependabotTypes.ts +++ b/extension/tasks/utils/dependabotTypes.ts @@ -103,5 +103,5 @@ export interface IDependabotUpdateOutput { } export interface IDependabotUpdateOutputProcessor { - process(type: string, data: any): Promise; + process(update: IDependabotUpdateJob, type: string, data: any): Promise; } diff --git a/extension/tasks/utils/dependabotUpdater.ts b/extension/tasks/utils/dependabotUpdater.ts index 94376765..d9ccc862 100644 --- a/extension/tasks/utils/dependabotUpdater.ts +++ b/extension/tasks/utils/dependabotUpdater.ts @@ -67,12 +67,14 @@ export class DependabotUpdater { } console.info("Running dependabot update..."); - const dependabotTool = tool(which("dependabot", true)).arg(dependabotArguments); - const dependabotResultCode = await dependabotTool.execAsync({ - silent: !this.debug - }); - if (dependabotResultCode != 0) { - throw new Error(`Dependabot update failed with exit code ${dependabotResultCode}`); + if (!fs.existsSync(jobOutputPath)) { + const dependabotTool = tool(which("dependabot", true)).arg(dependabotArguments); + const dependabotResultCode = await dependabotTool.execAsync({ + silent: !this.debug + }); + if (dependabotResultCode != 0) { + throw new Error(`Dependabot update failed with exit code ${dependabotResultCode}`); + } } console.info("Processing dependabot update outputs..."); @@ -89,7 +91,7 @@ export class DependabotUpdater { } }; try { - processedOutput.success = await this.outputProcessor.process(type, data); + processedOutput.success = await this.outputProcessor.process(config, type, data); } catch (e) { processedOutput.success = false; From 47e38637f55166b7dc70cb9b5de2823dc9add829 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Wed, 4 Sep 2024 00:02:56 +1200 Subject: [PATCH 08/57] Implement groups, auto-complete, auto-approve, pull request properties, and updating existing pull requests --- .../tasks/dependabot/dependabotV2/index.ts | 100 +++----- .../azure-devops/AzureDevOpsWebApiClient.ts | 212 +++++++++++++++++ .../azure-devops/interfaces/IPullRequest.ts | 42 ++++ .../interfaces/IPullRequestProperties.ts | 8 + extension/tasks/utils/azureDevOpsApiClient.ts | 133 ----------- .../azureDevOpsDependabotOutputProcessor.ts | 133 ----------- .../DependabotCli.ts} | 75 +++--- .../dependabot-cli/DependabotJobBuilder.ts | 117 ++++++++++ .../DependabotOutputProcessor.ts | 214 ++++++++++++++++++ .../interfaces/IDependabotUpdateJob.ts} | 67 +++--- .../interfaces/IDependabotUpdateOutput.ts | 9 + .../IDependabotUpdateOutputProcessor.ts | 5 + 12 files changed, 721 insertions(+), 394 deletions(-) create mode 100644 extension/tasks/utils/azure-devops/AzureDevOpsWebApiClient.ts create mode 100644 extension/tasks/utils/azure-devops/interfaces/IPullRequest.ts create mode 100644 extension/tasks/utils/azure-devops/interfaces/IPullRequestProperties.ts delete mode 100644 extension/tasks/utils/azureDevOpsApiClient.ts delete mode 100644 extension/tasks/utils/azureDevOpsDependabotOutputProcessor.ts rename extension/tasks/utils/{dependabotUpdater.ts => dependabot-cli/DependabotCli.ts} (68%) create mode 100644 extension/tasks/utils/dependabot-cli/DependabotJobBuilder.ts create mode 100644 extension/tasks/utils/dependabot-cli/DependabotOutputProcessor.ts rename extension/tasks/utils/{dependabotTypes.ts => dependabot-cli/interfaces/IDependabotUpdateJob.ts} (57%) create mode 100644 extension/tasks/utils/dependabot-cli/interfaces/IDependabotUpdateOutput.ts create mode 100644 extension/tasks/utils/dependabot-cli/interfaces/IDependabotUpdateOutputProcessor.ts diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index f0fd0f28..2cd2f4af 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -1,14 +1,15 @@ import { which, setResult, TaskResult } from "azure-pipelines-task-lib/task" import { debug, warning, error } from "azure-pipelines-task-lib/task" -import { IDependabotUpdateJob } from "../../utils/dependabotTypes"; -import { DependabotUpdater } from '../../utils/dependabotUpdater'; -import { AzureDevOpsClient } from "../../utils/azureDevOpsApiClient"; -import { AzureDevOpsDependabotOutputProcessor } from "../../utils/azureDevOpsDependabotOutputProcessor"; +import { DependabotCli } from '../../utils/dependabot-cli/DependabotCli'; +import { AzureDevOpsWebApiClient } from "../../utils/azure-devops/AzureDevOpsWebApiClient"; +import { DependabotOutputProcessor } from "../../utils/dependabot-cli/DependabotOutputProcessor"; +import { DependabotJobBuilder } from "../../utils/dependabot-cli/DependabotJobBuilder"; import { parseConfigFile } from '../../utils/parseConfigFile'; import getSharedVariables from '../../utils/getSharedVariables'; async function run() { - let dependabotCli: DependabotUpdater = undefined; + let taskWasSuccessful: boolean = true; + let dependabot: DependabotCli = undefined; try { // Check if required tools are installed @@ -22,78 +23,47 @@ async function run() { const dependabotConfig = await parseConfigFile(taskVariables); // Initialise the DevOps API client - const azdoApi = new AzureDevOpsClient( + const azdoApi = new AzureDevOpsWebApiClient( taskVariables.organizationUrl.toString(), taskVariables.systemAccessToken ); // Initialise the Dependabot updater - dependabotCli = new DependabotUpdater( - DependabotUpdater.CLI_IMAGE_LATEST, // TODO: Add config for this? - new AzureDevOpsDependabotOutputProcessor(azdoApi, taskVariables), + dependabot = new DependabotCli( + DependabotCli.CLI_IMAGE_LATEST, // TODO: Add config for this? + new DependabotOutputProcessor(azdoApi, taskVariables), taskVariables.debug ); - // Process the updates per-ecosystem - let taskWasSuccessful: boolean = true; - dependabotConfig.updates.forEach(async (update) => { - - // TODO: Fetch all existing PRs from DevOps + // Fetch all active Dependabot pull requests from DevOps + const myActivePullRequests = await azdoApi.getMyActivePullRequestProperties( + taskVariables.project, taskVariables.repository + ); - let registryCredentials = new Array(); - for (const key in dependabotConfig.registries) { - const registry = dependabotConfig.registries[key]; - registryCredentials.push({ - type: registry.type, - host: registry.host, - url: registry.url, - registry: registry.registry, - region: undefined, // TODO: registry.region, - username: registry.username, - password: registry.password, - token: registry.token, - 'replaces-base': registry['replaces-base'] || false + // Loop through each package ecyosystem and perform updates + dependabotConfig.updates.forEach(async (update) => { + const existingPullRequests = myActivePullRequests + .filter(pr => { + return pr.properties.find(p => p.name === DependabotOutputProcessor.PR_PROPERTY_NAME_PACKAGE_MANAGER && p.value === update.packageEcosystem); + }) + .map(pr => { + return JSON.parse( + pr.properties.find(p => p.name === DependabotOutputProcessor.PR_PROPERTY_NAME_DEPENDENCIES)?.value + ) }); - }; - let job: IDependabotUpdateJob = { - job: { - // TODO: Parse all options from `config` and `variables` - id: 'job-1', // TODO: Make timestamp or auto-incrementing id? - 'package-manager': update.packageEcosystem, - 'updating-a-pull-request': false, - 'allowed-updates': [ - { 'update-type': 'all' } // TODO: update.allow - ], - 'security-updates-only': false, - source: { - provider: 'azure', - 'api-endpoint': taskVariables.apiEndpointUrl, - hostname: taskVariables.hostname, - repo: `${taskVariables.organization}/${taskVariables.project}/_git/${taskVariables.repository}`, - branch: update.targetBranch, // TODO: add config for 'source branch'?? - commit: undefined, // TODO: add config for this? - directory: update.directories?.length == 0 ? update.directory : undefined, - directories: update.directories?.length > 0 ? update.directories : undefined - } - }, - credentials: (registryCredentials || []).concat([ - { - type: 'git_source', - host: taskVariables.hostname, - username: taskVariables.systemAccessUser?.trim()?.length > 0 ? taskVariables.systemAccessUser : 'x-access-token', - password: taskVariables.systemAccessToken - } - ]) - }; - - // Run dependabot updater for the job - if ((await dependabotCli.update(job)).filter(u => !u.success).length > 0) { + // Update all packages, this will update our dependencies list and create new pull requests + const allDependenciesJob = DependabotJobBuilder.updateAllDependenciesJob(taskVariables, update, dependabotConfig.registries, existingPullRequests); + if ((await dependabot.update(allDependenciesJob)).filter(u => !u.success).length > 0) { taskWasSuccessful = false; } - // TODO: Loop through all existing PRs and do a single update job for each, update/close the PR as needed - // e.g. https://github.com/dependabot/cli/blob/main/testdata/go/update-pr.yaml - + // Update existing pull requests, this will either resolve merge conflicts or close pull requests that are no longer needed + for (const pr of existingPullRequests) { + const updatePullRequestJob = DependabotJobBuilder.updatePullRequestJob(taskVariables, update, dependabotConfig.registries, existingPullRequests, pr); + if ((await dependabot.update(updatePullRequestJob)).filter(u => !u.success).length > 0) { + taskWasSuccessful = false; + } + } }); setResult( @@ -107,7 +77,7 @@ async function run() { setResult(TaskResult.Failed, e?.message); } finally { - dependabotCli?.cleanup(); + //dependabotCli?.cleanup(); } } diff --git a/extension/tasks/utils/azure-devops/AzureDevOpsWebApiClient.ts b/extension/tasks/utils/azure-devops/AzureDevOpsWebApiClient.ts new file mode 100644 index 00000000..e747495b --- /dev/null +++ b/extension/tasks/utils/azure-devops/AzureDevOpsWebApiClient.ts @@ -0,0 +1,212 @@ +import { debug, warning, error } from "azure-pipelines-task-lib/task" +import { WebApi, getPersonalAccessTokenHandler } from "azure-devops-node-api"; +import { ItemContentType, PullRequestStatus } from "azure-devops-node-api/interfaces/GitInterfaces"; +import { IPullRequestProperties } from "./interfaces/IPullRequestProperties"; +import { IPullRequest } from "./interfaces/IPullRequest"; + +// Wrapper for DevOps WebApi client with helper methods for easier management of dependabot pull requests +export class AzureDevOpsWebApiClient { + + private readonly connection: WebApi; + private userId: string | null = null; + + constructor(organisationApiUrl: string, accessToken: string) { + this.connection = new WebApi( + organisationApiUrl, + getPersonalAccessTokenHandler(accessToken) + ); + } + + // Get the properties for all active pull request created by the current user + public async getMyActivePullRequestProperties(project: string, repository: string): Promise { + console.info(`Fetching active pull request properties in '${project}/${repository}'...`); + try { + const userId = await this.getUserId(); + const git = await this.connection.getGitApi(); + const pullRequests = await git.getPullRequests( + repository, + { + creatorId: userId, + status: PullRequestStatus.Active + }, + project + ); + + return await Promise.all( + pullRequests.map(async pr => { + const properties = (await git.getPullRequestProperties(repository, pr.pullRequestId, project))?.value; + return { + id: pr.pullRequestId, + properties: Object.keys(properties)?.map(key => { + return { + name: key, + value: properties[key].$value + }; + }) || [] + }; + }) + ); + } + catch (e) { + error(`Failed to list active pull request properties: ${e}`); + return []; + } + } + + // Create a new pull request + public async createPullRequest(pr: IPullRequest): Promise { + console.info(`Creating pull request '${pr.title}'...`); + try { + const userId = await this.getUserId(); + const git = await this.connection.getGitApi(); + + // Create the source branch and commit the file changes + console.info(` - Pushing ${pr.changes.length} change(s) to branch '${pr.source.branch}'...`); + const push = await git.createPush( + { + refUpdates: [ + { + name: `refs/heads/${pr.source.branch}`, + oldObjectId: pr.source.commit + } + ], + commits: [ + { + comment: pr.commitMessage, + author: pr.author, + changes: pr.changes.map(change => { + return { + changeType: change.changeType, + item: { + path: normalizeDevOpsPath(change.path) + }, + newContent: { + content: change.content, + contentType: ItemContentType.RawText + } + }; + }) + } + ] + }, + pr.repository, + pr.project + ); + + // Create the pull request + console.info(` - Creating pull request to merge '${pr.source.branch}' into '${pr.target.branch}'...`); + const pullRequest = await git.createPullRequest( + { + sourceRefName: `refs/heads/${pr.source.branch}`, + targetRefName: `refs/heads/${pr.target.branch}`, + title: pr.title, + description: pr.description + }, + pr.repository, + pr.project, + true + ); + + // Add the pull request properties + if (pr.properties?.length > 0) { + console.info(` - Adding dependency metadata to pull request properties...`); + await git.updatePullRequestProperties( + null, + pr.properties.map(property => { + return { + op: "add", + path: "/" + property.name, + value: property.value + }; + }), + pr.repository, + pullRequest.pullRequestId, + pr.project + ); + } + + // Set the pull request auto-complete status + if (pr.autoComplete) { + console.info(` - Setting auto-complete...`); + const autoCompleteUserId = pr.autoComplete.userId || userId; + await git.updatePullRequest( + { + autoCompleteSetBy: { + id: autoCompleteUserId + }, + completionOptions: { + autoCompleteIgnoreConfigIds: pr.autoComplete.ignorePolicyConfigIds, + deleteSourceBranch: true, + mergeCommitMessage: mergeCommitMessage(pullRequest.pullRequestId, pr.title, pr.description), + mergeStrategy: pr.autoComplete.mergeStrategy, + transitionWorkItems: false, + } + }, + pr.repository, + pullRequest.pullRequestId, + pr.project + ); + } + + // Set the pull request auto-approve status + if (pr.autoApprove) { + console.info(` - Approving pull request...`); + const approverUserId = pr.autoApprove.userId || userId; + await git.createPullRequestReviewer( + { + vote: 10, // 10 - approved 5 - approved with suggestions 0 - no vote -5 - waiting for author -10 - rejected + isReapprove: true + }, + pr.repository, + pullRequest.pullRequestId, + approverUserId, + pr.project + ); + } + + console.info(` - Pull request ${pullRequest.pullRequestId} was created successfully.`); + return pullRequest.pullRequestId; + } + catch (e) { + error(`Failed to create pull request: ${e}`); + return null; + } + } + + private async getUserId(): Promise { + return (this.userId ||= (await this.connection.connect()).authenticatedUser?.id || ""); + } +} + +function normalizeDevOpsPath(path: string): string { + // Convert backslashes to forward slashes, convert './' => '/' and ensure the path starts with a forward slash if it doesn't already, this is how DevOps paths are formatted + return path.replace(/\\/g, "/").replace(/^\.\//, "/").replace(/^([^/])/, "/$1"); +} + +function mergeCommitMessage(id: number, title: string, description: string): string { + // + // Pull requests that pass all policies will be merged automatically. + // Optional policies can be ignored by passing their identifiers + // + // The merge commit message should contain the PR number and title for tracking. + // This is the default behaviour in Azure DevOps + // Example: + // Merged PR 24093: Bump Tingle.Extensions.Logging.LogAnalytics from 3.4.2-ci0005 to 3.4.2-ci0006 + // + // Bumps [Tingle.Extensions.Logging.LogAnalytics](...) from 3.4.2-ci0005 to 3.4.2-ci0006 + // - [Release notes](....) + // - [Changelog](....) + // - [Commits](....) + // + // There appears to be a DevOps bug when setting "completeOptions" with a "mergeCommitMessage" that is + // truncated to 4000 characters. The error message is: + // Invalid argument value. + // Parameter name: Completion options have exceeded the maximum encoded length (4184/4000) + // + // Most users seem to agree that the effective limit is about 3500 characters. + // https://developercommunity.visualstudio.com/t/raise-the-character-limit-for-pull-request-descrip/365708 + // + // Until this is fixed, we hard cap the max length to 3500 characters + // + return `Merged PR ${id}: ${title}\n\n${description}`.slice(0, 3500); +} \ No newline at end of file diff --git a/extension/tasks/utils/azure-devops/interfaces/IPullRequest.ts b/extension/tasks/utils/azure-devops/interfaces/IPullRequest.ts new file mode 100644 index 00000000..4660d4db --- /dev/null +++ b/extension/tasks/utils/azure-devops/interfaces/IPullRequest.ts @@ -0,0 +1,42 @@ +import { GitPullRequestMergeStrategy, VersionControlChangeType } from "azure-devops-node-api/interfaces/GitInterfaces"; + +export interface IPullRequest { + project: string, + repository: string, + source: { + commit: string, + branch: string + }, + target: { + branch: string + }, + author?: { + email: string, + name: string + }, + title: string, + description: string, + commitMessage: string, + autoComplete?: { + userId: string | undefined, + ignorePolicyConfigIds?: number[], + mergeStrategy?: GitPullRequestMergeStrategy + }, + autoApprove?: { + userId: string | undefined + }, + assignees?: string[], + reviewers?: string[], + labels?: string[], + workItems?: number[], + changes: { + changeType: VersionControlChangeType, + path: string, + content: string, + encoding: string + }[], + properties?: { + name: string, + value: string + }[] +}; diff --git a/extension/tasks/utils/azure-devops/interfaces/IPullRequestProperties.ts b/extension/tasks/utils/azure-devops/interfaces/IPullRequestProperties.ts new file mode 100644 index 00000000..bbc36433 --- /dev/null +++ b/extension/tasks/utils/azure-devops/interfaces/IPullRequestProperties.ts @@ -0,0 +1,8 @@ + +export interface IPullRequestProperties { + id: number, + properties?: { + name: string, + value: string + }[] +} diff --git a/extension/tasks/utils/azureDevOpsApiClient.ts b/extension/tasks/utils/azureDevOpsApiClient.ts deleted file mode 100644 index df293955..00000000 --- a/extension/tasks/utils/azureDevOpsApiClient.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { debug, warning, error } from "azure-pipelines-task-lib/task" -import { WebApi, getPersonalAccessTokenHandler } from "azure-devops-node-api"; -import { ItemContentType, VersionControlChangeType } from "azure-devops-node-api/interfaces/GitInterfaces"; - -export interface IPullRequest { - organisation: string, - project: string, - repository: string, - source: { - commit: string, - branch: string - }, - target: { - branch: string - }, - author: { - email: string, - name: string - }, - title: string, - description: string, - commitMessage: string, - mergeStrategy: string, - autoComplete: { - enabled: boolean, - bypassPolicyIds: number[], - }, - autoApprove: { - enabled: boolean, - approverUserToken: string - }, - assignees: string[], - reviewers: string[], - labels: string[], - workItems: number[], - dependencies: any, - changes: { - changeType: VersionControlChangeType, - path: string, - content: string, - encoding: string - }[] -}; - -// Wrapper for DevOps API actions -export class AzureDevOpsClient { - - private readonly connection: WebApi; - private userId: string | null = null; - - constructor(apiUrl: string, accessToken: string) { - this.connection = new WebApi( - apiUrl, - getPersonalAccessTokenHandler(accessToken) - ); - } - - public async createPullRequest(pr: IPullRequest): Promise { - console.info(`Creating pull request for '${pr.title}'...`); - try { - const userId = await this.getUserId(); - const git = await this.connection.getGitApi(); - - // Create a new branch for the pull request and commit the changes - console.info(`Pushing ${pr.changes.length} change(s) to branch '${pr.source.branch}'...`); - const push = await git.createPush( - { - refUpdates: [ - { - name: `refs/heads/${pr.source.branch}`, - oldObjectId: pr.source.commit - } - ], - commits: [ - { - comment: pr.commitMessage, - author: { - email: pr.author.email, - name: pr.author.name - }, - changes: pr.changes.map(change => { - return { - changeType: change.changeType, - item: { - path: normalizeDevOpsPath(change.path) - }, - newContent: { - content: change.content, - contentType: ItemContentType.RawText - } - }; - }) - } - ] - }, - pr.repository, - pr.project - ); - - // Create the pull request - console.info(`Creating pull request to merge '${pr.source.branch}' into '${pr.target.branch}'...`); - const pullRequest = await git.createPullRequest( - { - sourceRefName: `refs/heads/${pr.target.branch}`, // Merge from the new dependabot update branch - targetRefName: `refs/heads/${pr.source.branch}`, // Merge to the original branch - title: pr.title, - description: pr.description - - }, - pr.repository, - pr.project, - true - ); - - console.info(`Pull request #${pullRequest?.pullRequestId} created successfully.`); - return pullRequest?.pullRequestId || null; - } - catch (e) { - error(`Failed to create pull request: ${e}`); - console.log(e); - return null; - } - } - - private async getUserId(): Promise { - return (this.userId ||= (await this.connection.connect()).authenticatedUser?.id || ""); - } -} - -function normalizeDevOpsPath(path: string): string { - // Convert backslashes to forward slashes, convert './' => '/' and ensure the path starts with a forward slash if it doesn't already, this is how DevOps paths are formatted - return path.replace(/\\/g, "/").replace(/^\.\//, "/").replace(/^([^/])/, "/$1"); -} diff --git a/extension/tasks/utils/azureDevOpsDependabotOutputProcessor.ts b/extension/tasks/utils/azureDevOpsDependabotOutputProcessor.ts deleted file mode 100644 index c9c5dabc..00000000 --- a/extension/tasks/utils/azureDevOpsDependabotOutputProcessor.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { debug, warning, error } from "azure-pipelines-task-lib/task" -import { ISharedVariables } from "./getSharedVariables"; -import { IDependabotUpdateJob, IDependabotUpdateOutputProcessor } from "./dependabotTypes"; -import { AzureDevOpsClient } from "./azureDevOpsApiClient"; -import { VersionControlChangeType } from "azure-devops-node-api/interfaces/GitInterfaces"; -import * as path from 'path'; - -// Processes dependabot update outputs using the DevOps API -export class AzureDevOpsDependabotOutputProcessor implements IDependabotUpdateOutputProcessor { - private readonly api: AzureDevOpsClient; - private readonly taskVariables: ISharedVariables; - - constructor(api: AzureDevOpsClient, taskVariables: ISharedVariables) { - this.api = api; - this.taskVariables = taskVariables; - } - - // Process the appropriate DevOps API actions for the supplied dependabot update output - public async process(update: IDependabotUpdateJob, type: string, data: any): Promise { - console.debug(`Processing '${type}' with data:`, data); - let success: boolean = true; - switch (type) { - - case 'update_dependency_list': - // TODO: Store dependency list info in DevOps project data? - // This could be used to generate a dependency graph hub/page/report (future feature maybe?) - break; - - case 'create_pull_request': - if (this.taskVariables.skipPullRequests) { - warning(`Skipping pull request creation as 'skipPullRequests' is set to 'true'`); - return; - } - // TODO: Skip if active pull request limit reached. - - const sourceRepoParts = update.job.source.repo.split('/'); // "{organisation}/{project}/_git/{repository}"" - await this.api.createPullRequest({ - organisation: sourceRepoParts[0], - project: sourceRepoParts[1], - repository: sourceRepoParts[3], - source: { - commit: data['base-commit-sha'] || update.job.source.commit, - branch: `dependabot/${update.job["package-manager"]}/${update.job.id}` // TODO: Get from dependabot.yaml - }, - target: { - branch: 'main' // TODO: Get from dependabot.yaml - }, - author: { - email: 'noreply@github.com', // TODO: Get from task variables? - name: 'dependabot[bot]', // TODO: Get from task variables? - }, - title: data['pr-title'], - description: data['pr-body'], - commitMessage: data['commit-message'], - mergeStrategy: this.taskVariables.mergeStrategy, - autoComplete: { - enabled: this.taskVariables.setAutoComplete, - bypassPolicyIds: this.taskVariables.autoCompleteIgnoreConfigIds - }, - autoApprove: { - enabled: this.taskVariables.autoApprove, - approverUserToken: this.taskVariables.autoApproveUserToken - }, - assignees: [], // TODO: Get from dependabot.yaml - reviewers: [], // TODO: Get from dependabot.yaml - labels: [], // TODO: Get from dependabot.yaml - workItems: [], // TODO: Get from dependabot.yaml - dependencies: data['dependencies'], - changes: data['updated-dependency-files'].filter((file) => file['type'] === 'file').map((file) => { - let changeType = VersionControlChangeType.None; - if (file['deleted'] === true) { - changeType = VersionControlChangeType.Delete; - } else if (file['operation'] === 'update') { - changeType = VersionControlChangeType.Edit; - } else { - changeType = VersionControlChangeType.Add; - } - return { - changeType: changeType, - path: path.join(file['directory'], file['name']), - content: file['content'], - encoding: file['content_encoding'] - } - }) - }) - break; - - case 'update_pull_request': - if (this.taskVariables.skipPullRequests) { - warning(`Skipping pull request update as 'skipPullRequests' is set to 'true'`); - return; - } - // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: update_pull_request() - break; - - case 'close_pull_request': - if (this.taskVariables.abandonUnwantedPullRequests) { - warning(`Skipping pull request closure as 'abandonUnwantedPullRequests' is set to 'true'`); - return; - } - // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: close_pull_request() - break; - - case 'mark_as_processed': - // TODO: Log this? - break; - - case 'record_ecosystem_versions': - // TODO: Log this? - break; - - case 'record_update_job_error': - // TODO: Log this? - success = false; - break; - - case 'record_update_job_unknown_error': - // TODO: Log this? - success = false; - break; - - case 'increment_metric': - // TODO: Log this? - break; - - default: - warning(`Unknown dependabot output type '${type}', ignoring...`); - break; - } - - return success; - } -} \ No newline at end of file diff --git a/extension/tasks/utils/dependabotUpdater.ts b/extension/tasks/utils/dependabot-cli/DependabotCli.ts similarity index 68% rename from extension/tasks/utils/dependabotUpdater.ts rename to extension/tasks/utils/dependabot-cli/DependabotCli.ts index d9ccc862..5daebe29 100644 --- a/extension/tasks/utils/dependabotUpdater.ts +++ b/extension/tasks/utils/dependabot-cli/DependabotCli.ts @@ -1,14 +1,16 @@ import { debug, warning, error } from "azure-pipelines-task-lib/task" import { which, tool } from "azure-pipelines-task-lib/task" import { ToolRunner } from "azure-pipelines-task-lib/toolrunner" -import { IDependabotUpdateOutputProcessor, IDependabotUpdateJob, IDependabotUpdateOutput } from "./dependabotTypes"; +import { IDependabotUpdateOutputProcessor } from "./interfaces/IDependabotUpdateOutputProcessor"; +import { IDependabotUpdateOutput } from "./interfaces/IDependabotUpdateOutput"; +import { IDependabotUpdateJob } from "./interfaces/IDependabotUpdateJob"; import * as yaml from 'js-yaml'; import * as path from 'path'; import * as fs from 'fs'; import * as os from 'os'; // Wrapper class for running updates using dependabot-cli -export class DependabotUpdater { +export class DependabotCli { private readonly jobsPath: string; private readonly toolImage: string; private readonly outputProcessor: IDependabotUpdateOutputProcessor; @@ -24,7 +26,7 @@ export class DependabotUpdater { this.ensureJobsPathExists(); } - // Run dependabot update + // Run dependabot update job public async update( config: IDependabotUpdateJob, options?: { @@ -47,9 +49,6 @@ export class DependabotUpdater { fs.mkdirSync(jobPath); } - // Generate the job input file - writeJobInput(jobInputPath, config); - // Compile dependabot cmd arguments // See: https://github.com/dependabot/cli/blob/main/cmd/dependabot/internal/cmd/root.go // https://github.com/dependabot/cli/blob/main/cmd/dependabot/internal/cmd/update.go @@ -66,8 +65,12 @@ export class DependabotUpdater { dependabotArguments.push("--updater-image", options.updaterImage); } - console.info("Running dependabot update..."); + // Generate the job input file + writeJobFile(jobInputPath, config); + + // Run dependabot update if (!fs.existsSync(jobOutputPath)) { + console.info("Running dependabot update..."); const dependabotTool = tool(which("dependabot", true)).arg(dependabotArguments); const dependabotResultCode = await dependabotTool.execAsync({ silent: !this.debug @@ -77,28 +80,33 @@ export class DependabotUpdater { } } - console.info("Processing dependabot update outputs..."); + // Process the job output const processedOutputs = Array(); - for (const output of readScenarioOutputs(jobOutputPath)) { - const type = output['type']; - const data = output['expect']?.['data']; - var processedOutput = { - success: true, - error: null, - output: { - type: type, - data: data + if (fs.existsSync(jobOutputPath)) { + console.info("Processing dependabot update outputs..."); + for (const output of readScenarioOutputs(jobOutputPath)) { + // Documentation on the scenario model can be found here: + // https://github.com/dependabot/cli/blob/main/internal/model/scenario.go + const type = output['type']; + const data = output['expect']?.['data']; + var processedOutput = { + success: true, + error: null, + output: { + type: type, + data: data + } + }; + try { + processedOutput.success = await this.outputProcessor.process(config, type, data); + } + catch (e) { + processedOutput.success = false; + processedOutput.error = e; + } + finally { + processedOutputs.push(processedOutput); } - }; - try { - processedOutput.success = await this.outputProcessor.process(config, type, data); - } - catch (e) { - processedOutput.success = false; - processedOutput.error = e; - } - finally { - processedOutputs.push(processedOutput); } } @@ -113,7 +121,7 @@ export class DependabotUpdater { return; } - debug("Dependabot CLI was not found, installing with `go install`..."); + console.info("Dependabot CLI install was not found, installing now with `go install`..."); const goTool: ToolRunner = tool(which("go", true)); goTool.arg(["install", this.toolImage]); goTool.execSync({ @@ -139,10 +147,17 @@ export class DependabotUpdater { } } -function writeJobInput(path: string, config: IDependabotUpdateJob): void { - fs.writeFileSync(path, yaml.dump(config)); +// Documentation on the job model can be found here: +// https://github.com/dependabot/cli/blob/main/internal/model/job.go +function writeJobFile(path: string, config: IDependabotUpdateJob): void { + fs.writeFileSync(path, yaml.dump({ + job: config.job, + credentials: config.credentials + })); } +// Documentation on the scenario model can be found here: +// https://github.com/dependabot/cli/blob/main/internal/model/scenario.go function readScenarioOutputs(path: string): any[] { if (!path) { throw new Error("Scenario file path is required"); diff --git a/extension/tasks/utils/dependabot-cli/DependabotJobBuilder.ts b/extension/tasks/utils/dependabot-cli/DependabotJobBuilder.ts new file mode 100644 index 00000000..4c251496 --- /dev/null +++ b/extension/tasks/utils/dependabot-cli/DependabotJobBuilder.ts @@ -0,0 +1,117 @@ +import { ISharedVariables } from "../getSharedVariables"; +import { IDependabotRegistry, IDependabotUpdate } from "../IDependabotConfig"; +import { IDependabotUpdateJob } from "./interfaces/IDependabotUpdateJob"; + +// Wrapper class for building dependabot update job objects +export class DependabotJobBuilder { + + // Create a dependabot update job that updates all dependencies for a package ecyosystem + public static updateAllDependenciesJob( + variables: ISharedVariables, + update: IDependabotUpdate, + registries: Record, + existingPullRequests: any[] + ): IDependabotUpdateJob { + return { + config: update, + job: { + // TODO: Parse all options from `config` and `variables` + id: `update-${update.packageEcosystem}-all`, // TODO: Refine this + 'package-manager': update.packageEcosystem, + 'update-subdependencies': true, // TODO: add config option + 'updating-a-pull-request': false, + 'dependency-groups': mapDependencyGroups(update.groups), + 'allowed-updates': [ + { 'update-type': 'all' } // TODO: update.allow + ], + 'ignore-conditions': [], // TODO: update.ignore + 'security-updates-only': false, // TODO: update.'security-updates-only' + 'security-advisories': [], // TODO: update.securityAdvisories + source: { + provider: 'azure', + 'api-endpoint': variables.apiEndpointUrl, + hostname: variables.hostname, + repo: `${variables.organization}/${variables.project}/_git/${variables.repository}`, + branch: update.targetBranch, + commit: undefined, // use latest commit of target branch + directory: update.directories?.length == 0 ? update.directory : undefined, + directories: update.directories?.length > 0 ? update.directories : undefined + }, + 'existing-pull-requests': existingPullRequests.filter(pr => !pr['dependency-group-name']), + 'existing-group-pull-requests': existingPullRequests.filter(pr => pr['dependency-group-name']), + 'commit-message-options': undefined, // TODO: update.commitMessageOptions + 'experiments': undefined, // TODO: update.experiments + 'max-updater-run-time': undefined, // TODO: update.maxUpdaterRunTime + 'reject-external-code': undefined, // TODO: update.insecureExternalCodeExecution + 'repo-private': undefined, // TODO: update.repoPrivate + 'repo-contents-path': undefined, // TODO: update.repoContentsPath + 'requirements-update-strategy': undefined, // TODO: update.requirementsUpdateStrategy + 'lockfile-only': undefined, // TODO: update.lockfileOnly + 'vendor-dependencies': undefined, // TODO: update.vendorDependencies + 'debug': variables.debug + }, + credentials: mapRegistryCredentials(variables, registries) + }; + } + + // Create a dependabot update job that updates a single pull request + static updatePullRequestJob( + variables: ISharedVariables, + update: IDependabotUpdate, + registries: Record, + existingPullRequests: any[], + pullRequestToUpdate: any + ): IDependabotUpdateJob { + const dependencyGroupName = pullRequestToUpdate['dependency-group-name']; + const dependencies = (dependencyGroupName ? pullRequestToUpdate['dependencies'] : pullRequestToUpdate)?.map(d => d['dependency-name']); + const result = this.updateAllDependenciesJob(variables, update, registries, existingPullRequests); + result.job['id'] = `update-${update.packageEcosystem}-${Date.now()}`; // TODO: Refine this + result.job['updating-a-pull-request'] = true; + result.job['dependency-group-to-refresh'] = dependencyGroupName; + result.job['dependencies'] = dependencies; + return result; + } + +} + +// Map registry credentials +function mapRegistryCredentials(variables: ISharedVariables, registries: Record): any[] { + let registryCredentials = new Array(); + if (variables.systemAccessToken) { + registryCredentials.push({ + type: 'git_source', + host: variables.hostname, + username: variables.systemAccessUser?.trim()?.length > 0 ? variables.systemAccessUser : 'x-access-token', + password: variables.systemAccessToken + }); + } + if (registries) { + for (const key in registries) { + const registry = registries[key]; + registryCredentials.push({ + type: registry.type, + host: registry.host, + url: registry.url, + registry: registry.registry, + region: undefined, // TODO: registry.region, + username: registry.username, + password: registry.password, + token: registry.token, + 'replaces-base': registry['replaces-base'] || false + }); + } + } + + return registryCredentials; +} + +// Map dependency groups +function mapDependencyGroups(groups: string): any[] { + const dependencyGroups = JSON.parse(groups); + return Object.keys(dependencyGroups).map(name => { + return { + 'name': name, + 'rules': dependencyGroups[name] + }; + }); +} diff --git a/extension/tasks/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/utils/dependabot-cli/DependabotOutputProcessor.ts new file mode 100644 index 00000000..845fb942 --- /dev/null +++ b/extension/tasks/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -0,0 +1,214 @@ +import { debug, warning, error } from "azure-pipelines-task-lib/task" +import { ISharedVariables } from "../getSharedVariables"; +import { IDependabotUpdateJob } from "./interfaces/IDependabotUpdateJob"; +import { IDependabotUpdateOutputProcessor } from "./interfaces/IDependabotUpdateOutputProcessor"; +import { AzureDevOpsWebApiClient } from "../azure-devops/AzureDevOpsWebApiClient"; +import { GitPullRequestMergeStrategy, VersionControlChangeType } from "azure-devops-node-api/interfaces/GitInterfaces"; +import * as path from 'path'; +import * as crypto from 'crypto'; + +// Processes dependabot update outputs using the DevOps API +export class DependabotOutputProcessor implements IDependabotUpdateOutputProcessor { + private readonly api: AzureDevOpsWebApiClient; + private readonly taskVariables: ISharedVariables; + + // Custom properties used to store dependabot metadata in pull requests. + // https://learn.microsoft.com/en-us/rest/api/azure/devops/git/pull-request-properties + public static PR_PROPERTY_NAME_PACKAGE_MANAGER = "Dependabot.PackageManager"; + public static PR_PROPERTY_NAME_DEPENDENCIES = "Dependabot.Dependencies"; + + constructor(api: AzureDevOpsWebApiClient, taskVariables: ISharedVariables) { + this.api = api; + this.taskVariables = taskVariables; + } + + // Process the appropriate DevOps API actions for the supplied dependabot update output + public async process(update: IDependabotUpdateJob, type: string, data: any): Promise { + console.debug(`Processing output '${type}' with data:`, data); + let success: boolean = true; + switch (type) { + + // Documentation on the 'data' model for each output type can be found here: + // See: https://github.com/dependabot/cli/blob/main/internal/model/update.go + + case 'update_dependency_list': + // TODO: Store dependency list info in DevOps project properties? + // This could be used to generate a dependency graph hub/page/report (future feature maybe?) + // https://learn.microsoft.com/en-us/rest/api/azure/devops/core/projects/set-project-properties + break; + + case 'create_pull_request': + if (this.taskVariables.skipPullRequests) { + warning(`Skipping pull request creation as 'skipPullRequests' is set to 'true'`); + return; + } + + // TODO: Skip if active pull request limit reached. + + const sourceRepoParts = update.job.source.repo.split('/'); // "{organisation}/{project}/_git/{repository}"" + const dependencyGroupName = data['dependency-group']?.['name']; + let dependencies: any = data['dependencies']?.map((dep) => { + return { + 'dependency-name': dep['name'], + 'dependency-version': dep['version'], + 'directory': dep['directory'], + }; + }); + if (dependencyGroupName) { + dependencies = { + 'dependency-group-name': dependencyGroupName, + 'dependencies': dependencies + }; + } + await this.api.createPullRequest({ + project: sourceRepoParts[1], + repository: sourceRepoParts[3], + source: { + commit: data['base-commit-sha'] || update.job.source.commit, + branch: sourceBranchNameForUpdate(update.job["package-manager"], update.config.targetBranch, dependencies) + }, + target: { + branch: update.config.targetBranch + }, + author: { + email: 'noreply@github.com', // TODO: this.taskVariables.extraEnvironmentVariables['DEPENDABOT_AUTHOR_EMAIL'] + name: 'dependabot[bot]', // TODO: this.taskVariables.extraEnvironmentVariables['DEPENDABOT_AUTHOR_NAME'] + }, + title: data['pr-title'], + description: data['pr-body'], + commitMessage: data['commit-message'], + autoComplete: this.taskVariables.setAutoComplete ? { + userId: undefined, // TODO: add config for this? + ignorePolicyConfigIds: this.taskVariables.autoCompleteIgnoreConfigIds, + mergeStrategy: GitPullRequestMergeStrategy[this.taskVariables.mergeStrategy as keyof typeof GitPullRequestMergeStrategy] + } : undefined, + autoApprove: this.taskVariables.autoApprove ? { + userId: this.taskVariables.autoApproveUserToken // TODO: convert token to user id + } : undefined, + assignees: update.config.assignees, + reviewers: update.config.reviewers, + labels: update.config.labels?.split(',').map((label) => label.trim()) || [], + workItems: update.config.milestone ? [Number(update.config.milestone)] : [], + changes: data['updated-dependency-files'].filter((file) => file['type'] === 'file').map((file) => { + let changeType = VersionControlChangeType.None; + if (file['deleted'] === true) { + changeType = VersionControlChangeType.Delete; + } else if (file['operation'] === 'update') { + changeType = VersionControlChangeType.Edit; + } else { + changeType = VersionControlChangeType.Add; + } + return { + changeType: changeType, + path: path.join(file['directory'], file['name']), + content: file['content'], + encoding: file['content_encoding'] + } + }), + properties: [ + { + name: DependabotOutputProcessor.PR_PROPERTY_NAME_PACKAGE_MANAGER, + value: update.job["package-manager"] + }, + { + name: DependabotOutputProcessor.PR_PROPERTY_NAME_DEPENDENCIES, + value: JSON.stringify(dependencies) + } + ] + }) + break; + + case 'update_pull_request': + if (this.taskVariables.skipPullRequests) { + warning(`Skipping pull request update as 'skipPullRequests' is set to 'true'`); + return; + } + // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: update_pull_request() + /* + type UpdatePullRequest struct { + BaseCommitSha string `json:"base-commit-sha" yaml:"base-commit-sha"` + DependencyNames []string `json:"dependency-names" yaml:"dependency-names"` + UpdatedDependencyFiles []DependencyFile `json:"updated-dependency-files" yaml:"updated-dependency-files"` + PRTitle string `json:"pr-title" yaml:"pr-title,omitempty"` + PRBody string `json:"pr-body" yaml:"pr-body,omitempty"` + CommitMessage string `json:"commit-message" yaml:"commit-message,omitempty"` + DependencyGroup map[string]any `json:"dependency-group" yaml:"dependency-group,omitempty"` + } + type DependencyFile struct { + Content string `json:"content" yaml:"content"` + ContentEncoding string `json:"content_encoding" yaml:"content_encoding"` + Deleted bool `json:"deleted" yaml:"deleted"` + Directory string `json:"directory" yaml:"directory"` + Name string `json:"name" yaml:"name"` + Operation string `json:"operation" yaml:"operation"` + SupportFile bool `json:"support_file" yaml:"support_file"` + SymlinkTarget string `json:"symlink_target,omitempty" yaml:"symlink_target,omitempty"` + Type string `json:"type" yaml:"type"` + Mode string `json:"mode" yaml:"mode,omitempty"` + } + */ + break; + + case 'close_pull_request': + if (this.taskVariables.abandonUnwantedPullRequests) { + warning(`Skipping pull request closure as 'abandonUnwantedPullRequests' is set to 'true'`); + return; + } + // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: close_pull_request() + /* + type ClosePullRequest struct { + DependencyNames []string `json:"dependency-names" yaml:"dependency-names"` + Reason string `json:"reason" yaml:"reason"` + } + */ + break; + + case 'mark_as_processed': + // No action required + break; + + case 'record_ecosystem_versions': + // No action required + break; + + case 'record_update_job_error': + error(`Update job error: ${data['error-type']}`); + console.log(data['error-details']); + success = false; + break; + + case 'record_update_job_unknown_error': + error(`Update job unknown error: ${data['error-type']}`); + console.log(data['error-details']); + success = false; + break; + + case 'increment_metric': + // No action required + break; + + default: + warning(`Unknown dependabot output type '${type}', ignoring...`); + break; + } + + return success; + } +} + +function sourceBranchNameForUpdate(packageEcosystem: string, targetBranch: string, dependencies: any): string { + const target = targetBranch?.replace(/^\/+|\/+$/g, ''); // strip leading/trailing slashes + if (dependencies['dependency-group-name']) { + // Group dependency update + // e.g. dependabot/nuget/main/microsoft-3b49c54d9e + const dependencyGroupName = dependencies['dependency-group-name']; + const dependencyHash = crypto.createHash('md5').update(dependencies['dependencies'].map(d => `${d['dependency-name']}-${d['dependency-version']}`).join(',')).digest('hex').substring(0, 10); + return `dependabot/${packageEcosystem}/${target}/${dependencyGroupName}-${dependencyHash}`; + } + else { + // Single dependency update + // e.g. dependabot/nuget/main/Microsoft.Extensions.Logging-1.0.0 + const leadDependency = dependencies.length === 1 ? dependencies[0] : null; + return `dependabot/${packageEcosystem}/${target}/${leadDependency['dependency-name']}-${leadDependency['dependency-version']}`; + } +} diff --git a/extension/tasks/utils/dependabotTypes.ts b/extension/tasks/utils/dependabot-cli/interfaces/IDependabotUpdateJob.ts similarity index 57% rename from extension/tasks/utils/dependabotTypes.ts rename to extension/tasks/utils/dependabot-cli/interfaces/IDependabotUpdateJob.ts index 31eba762..0e9c5443 100644 --- a/extension/tasks/utils/dependabotTypes.ts +++ b/extension/tasks/utils/dependabot-cli/interfaces/IDependabotUpdateJob.ts @@ -1,19 +1,28 @@ +import { IDependabotUpdate } from "../../IDependabotConfig" export interface IDependabotUpdateJob { + + // The raw dependabot.yaml update configuration options + // See: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + config: IDependabotUpdate, + + // The dependabot "updater" job configuration + // See: https://github.com/dependabot/cli/blob/main/internal/model/job.go + // https://github.com/dependabot/dependabot-core/blob/main/updater/lib/dependabot/job.rb job: { - // See: https://github.com/dependabot/dependabot-core/blob/main/updater/lib/dependabot/job.rb 'id': string, 'package-manager': string, - 'updating-a-pull-request': boolean, + 'update-subdependencies'?: boolean, + 'updating-a-pull-request'?: boolean, 'dependency-group-to-refresh'?: string, 'dependency-groups'?: { 'name': string, 'applies-to'?: string, - 'update-types'?: string[], 'rules': { 'patterns'?: string[] 'exclude-patterns'?: string[], 'dependency-type'?: string + 'update-types'?: string[] }[] }[], 'dependencies'?: string[], @@ -24,9 +33,12 @@ export interface IDependabotUpdateJob { }[], 'ignore-conditions'?: { 'dependency-name'?: string, - 'version-requirement'?: string, 'source'?: string, - 'update-types'?: string[] + 'update-types'?: string[], + // TODO: 'updated-at' config is not in the dependabot.yaml docs, but is in the dependabot-cli and dependabot-core models + // https://github.com/dependabot/dependabot-core/blob/dc7b8d2ad152780e847ab597d40a57f13ab86d4f/common/lib/dependabot/pull_request_creator/message_builder.rb#L762 + 'updated-at'?: string, + 'version-requirement'?: string, }[], 'security-updates-only': boolean, 'security-advisories'?: { @@ -34,6 +46,7 @@ export interface IDependabotUpdateJob { 'affected-versions': string[], 'patched-versions': string[], 'unaffected-versions': string[], + // TODO: The below configs are not in the dependabot-cli model, but are in the dependabot-core model 'title'?: string, 'description'?: string, 'source-name'?: string, @@ -50,36 +63,37 @@ export interface IDependabotUpdateJob { 'directories'?: string[] }, 'existing-pull-requests'?: { - 'dependencies': { - 'name': string, - 'version'?: string, - 'removed': boolean, - 'directory'?: string - }[] - }, + 'dependency-name': string, + 'dependency-version': string, + 'directory': string + }[][], 'existing-group-pull-requests'?: { 'dependency-group-name': string, 'dependencies': { - 'name': string, - 'version'?: string, - 'removed': boolean, - 'directory'?: string + 'dependency-name': string, + 'dependency-version': string, + 'directory': string }[] - }, + }[], 'commit-message-options'?: { 'prefix'?: string, 'prefix-development'?: string, - 'include'?: string, + 'include-scope'?: string, }, 'experiments'?: any, 'max-updater-run-time'?: number, 'reject-external-code'?: boolean, + 'repo-private'?: boolean, 'repo-contents-path'?: string, 'requirements-update-strategy'?: string, - 'lockfile-only'?: boolean + 'lockfile-only'?: boolean, + 'vendor-dependencies'?: boolean, + 'debug'?: boolean, }, + + // The dependabot "proxy" registry credentials + // See: https://github.com/dependabot/dependabot-core/blob/main/common/lib/dependabot/credential.rb credentials: { - // See: https://github.com/dependabot/dependabot-core/blob/main/common/lib/dependabot/credential.rb 'type': string, 'host'?: string, 'url'?: string, @@ -90,18 +104,5 @@ export interface IDependabotUpdateJob { 'token'?: string, 'replaces-base'?: boolean }[] -} - -export interface IDependabotUpdateOutput { - success: boolean, - error: any, - output: { - // See: https://github.com/dependabot/smoke-tests/tree/main/tests - type: string, - data: any - } -} -export interface IDependabotUpdateOutputProcessor { - process(update: IDependabotUpdateJob, type: string, data: any): Promise; } diff --git a/extension/tasks/utils/dependabot-cli/interfaces/IDependabotUpdateOutput.ts b/extension/tasks/utils/dependabot-cli/interfaces/IDependabotUpdateOutput.ts new file mode 100644 index 00000000..972a25e7 --- /dev/null +++ b/extension/tasks/utils/dependabot-cli/interfaces/IDependabotUpdateOutput.ts @@ -0,0 +1,9 @@ + +export interface IDependabotUpdateOutput { + success: boolean, + error: any, + output: { + type: string, + data: any + } +} diff --git a/extension/tasks/utils/dependabot-cli/interfaces/IDependabotUpdateOutputProcessor.ts b/extension/tasks/utils/dependabot-cli/interfaces/IDependabotUpdateOutputProcessor.ts new file mode 100644 index 00000000..df355221 --- /dev/null +++ b/extension/tasks/utils/dependabot-cli/interfaces/IDependabotUpdateOutputProcessor.ts @@ -0,0 +1,5 @@ +import { IDependabotUpdateJob } from "./IDependabotUpdateJob"; + +export interface IDependabotUpdateOutputProcessor { + process(update: IDependabotUpdateJob, type: string, data: any): Promise; +} From ce5b5ccf8deeabcd94b6d280fdf41bdba151316a Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Wed, 4 Sep 2024 00:26:19 +1200 Subject: [PATCH 09/57] Clean up --- extension/tasks/dependabot/dependabotV2/index.ts | 4 ++-- .../utils/azure-devops/AzureDevOpsWebApiClient.ts | 15 +++++---------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index 2cd2f4af..fe3768e5 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -51,7 +51,7 @@ async function run() { ) }); - // Update all packages, this will update our dependencies list and create new pull requests + // Update all dependencies, this will update create new pull requests const allDependenciesJob = DependabotJobBuilder.updateAllDependenciesJob(taskVariables, update, dependabotConfig.registries, existingPullRequests); if ((await dependabot.update(allDependenciesJob)).filter(u => !u.success).length > 0) { taskWasSuccessful = false; @@ -77,7 +77,7 @@ async function run() { setResult(TaskResult.Failed, e?.message); } finally { - //dependabotCli?.cleanup(); + // TODO: dependabotCli?.cleanup(); } } diff --git a/extension/tasks/utils/azure-devops/AzureDevOpsWebApiClient.ts b/extension/tasks/utils/azure-devops/AzureDevOpsWebApiClient.ts index e747495b..32b5f31e 100644 --- a/extension/tasks/utils/azure-devops/AzureDevOpsWebApiClient.ts +++ b/extension/tasks/utils/azure-devops/AzureDevOpsWebApiClient.ts @@ -184,12 +184,9 @@ function normalizeDevOpsPath(path: string): string { } function mergeCommitMessage(id: number, title: string, description: string): string { - // - // Pull requests that pass all policies will be merged automatically. - // Optional policies can be ignored by passing their identifiers // // The merge commit message should contain the PR number and title for tracking. - // This is the default behaviour in Azure DevOps + // This is the default behaviour in Azure DevOps. // Example: // Merged PR 24093: Bump Tingle.Extensions.Logging.LogAnalytics from 3.4.2-ci0005 to 3.4.2-ci0006 // @@ -198,15 +195,13 @@ function mergeCommitMessage(id: number, title: string, description: string): str // - [Changelog](....) // - [Commits](....) // - // There appears to be a DevOps bug when setting "completeOptions" with a "mergeCommitMessage" that is - // truncated to 4000 characters. The error message is: + // There appears to be a DevOps bug when setting "completeOptions" with a "mergeCommitMessage" even when truncated to 4000 characters. + // The error message is: // Invalid argument value. // Parameter name: Completion options have exceeded the maximum encoded length (4184/4000) // - // Most users seem to agree that the effective limit is about 3500 characters. - // https://developercommunity.visualstudio.com/t/raise-the-character-limit-for-pull-request-descrip/365708 - // - // Until this is fixed, we hard cap the max length to 3500 characters + // The effective limit seems to be about 3500 characters: + // https://developercommunity.visualstudio.com/t/raise-the-character-limit-for-pull-request-descrip/365708#T-N424531 // return `Merged PR ${id}: ${title}\n\n${description}`.slice(0, 3500); } \ No newline at end of file From 81728caf3385ad52265122a0b6c7c4330e4b398a Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Wed, 4 Sep 2024 00:30:20 +1200 Subject: [PATCH 10/57] Set task V1 as deprecated, task V2 as preview --- extension/tasks/dependabot/dependabotV1/task.json | 2 ++ extension/tasks/dependabot/dependabotV2/task.json | 1 + 2 files changed, 3 insertions(+) diff --git a/extension/tasks/dependabot/dependabotV1/task.json b/extension/tasks/dependabot/dependabotV1/task.json index 364cc258..e608c69f 100644 --- a/extension/tasks/dependabot/dependabotV1/task.json +++ b/extension/tasks/dependabot/dependabotV1/task.json @@ -17,6 +17,8 @@ "Minor": 33, "Patch": 0 }, + "deprecated": true, + "deprecationMessage": "This task version is deprecated due to updates in dependabot-core. The recommended approach to running dependabot has changed, and this task is no longer maintained. Please upgrade to the latest version to continue receiving fixes and features. More details: https://github.com/tinglesoftware/dependabot-azure-devops/discussions/1317.", "instanceNameFormat": "Dependabot", "minimumAgentVersion": "3.232.1", "groups": [ diff --git a/extension/tasks/dependabot/dependabotV2/task.json b/extension/tasks/dependabot/dependabotV2/task.json index fd494c4b..ef09db0e 100644 --- a/extension/tasks/dependabot/dependabotV2/task.json +++ b/extension/tasks/dependabot/dependabotV2/task.json @@ -17,6 +17,7 @@ "Minor": 0, "Patch": 0 }, + "preview": true, "instanceNameFormat": "Dependabot", "minimumAgentVersion": "3.232.1", "groups": [ From ac81a8891ce23660b7d1f53fd39ee7c5e0d2247c Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Wed, 4 Sep 2024 10:37:37 +1200 Subject: [PATCH 11/57] Restructure extension task to better support multiple version --- .github/workflows/extension.yml | 12 ++++++------ docs/extension.md | 12 +++++++++--- extension/.gitignore | 2 +- extension/overrides.local.json | 2 +- extension/package.json | 4 +++- .../dependabot/dependabotV1}/icon.png | Bin .../dependabot/dependabotV1}/index.ts | 8 ++++---- .../dependabot/dependabotV1}/task.json | 2 +- .../{task => tasks/utils}/IDependabotConfig.ts | 0 .../{task => tasks}/utils/convertPlaceholder.ts | 0 extension/{task => tasks}/utils/extractHostname.ts | 0 .../{task => tasks}/utils/extractOrganization.ts | 0 .../utils/extractVirtualDirectory.ts | 0 .../utils/getAzureDevOpsAccessToken.ts | 0 .../{task => tasks}/utils/getDockerImageTag.ts | 0 .../{task => tasks}/utils/getGithubAccessToken.ts | 0 .../{task => tasks}/utils/getSharedVariables.ts | 0 extension/{task => tasks}/utils/parseConfigFile.ts | 2 +- .../utils/resolveAzureDevOpsIdentities.ts | 0 extension/tests/utils/convertPlaceholder.test.ts | 2 +- extension/tests/utils/extractHostname.test.ts | 2 +- extension/tests/utils/extractOrganization.test.ts | 2 +- .../tests/utils/extractVirtualDirectory.test.ts | 2 +- extension/tests/utils/parseConfigFile.test.ts | 4 ++-- .../utils/resolveAzureDevOpsIdentities.test.ts | 2 +- extension/vss-extension.json | 4 ++-- 26 files changed, 35 insertions(+), 27 deletions(-) rename extension/{task => tasks/dependabot/dependabotV1}/icon.png (100%) rename extension/{task => tasks/dependabot/dependabotV1}/index.ts (97%) rename extension/{task => tasks/dependabot/dependabotV1}/task.json (99%) rename extension/{task => tasks/utils}/IDependabotConfig.ts (100%) rename extension/{task => tasks}/utils/convertPlaceholder.ts (100%) rename extension/{task => tasks}/utils/extractHostname.ts (100%) rename extension/{task => tasks}/utils/extractOrganization.ts (100%) rename extension/{task => tasks}/utils/extractVirtualDirectory.ts (100%) rename extension/{task => tasks}/utils/getAzureDevOpsAccessToken.ts (100%) rename extension/{task => tasks}/utils/getDockerImageTag.ts (100%) rename extension/{task => tasks}/utils/getGithubAccessToken.ts (100%) rename extension/{task => tasks}/utils/getSharedVariables.ts (100%) rename extension/{task => tasks}/utils/parseConfigFile.ts (99%) rename extension/{task => tasks}/utils/resolveAzureDevOpsIdentities.ts (100%) diff --git a/.github/workflows/extension.yml b/.github/workflows/extension.yml index c52029ab..b7ca1bfc 100644 --- a/.github/workflows/extension.yml +++ b/.github/workflows/extension.yml @@ -60,13 +60,13 @@ jobs: working-directory: '${{ github.workspace }}/extension' - name: Build - run: npm run build:prod + run: npm run build working-directory: '${{ github.workspace }}/extension' - name: Install tfx-cli run: npm install -g tfx-cli@0.12.0 - - name: Replace tokens + - name: Update version numbers in vss-extension.json overrides files uses: cschleiden/replace-tokens@v1 with: files: '["${{ github.workspace }}/extension/overrides*.json"]' @@ -74,11 +74,11 @@ jobs: MAJOR_MINOR_PATCH: ${{ steps.gitversion.outputs.majorMinorPatch }} BUILD_NUMBER: ${{ github.run_number }} - - name: Update values in extension/task/task.json + - name: Update version numbers in task.json run: | - echo "`jq '.version.Major=${{ steps.gitversion.outputs.major }}' extension/task/task.json`" > extension/task/task.json - echo "`jq '.version.Minor=${{ steps.gitversion.outputs.minor }}' extension/task/task.json`" > extension/task/task.json - echo "`jq '.version.Patch=${{ github.run_number }}' extension/task/task.json`" > extension/task/task.json + echo "`jq '.version.Major=${{ steps.gitversion.outputs.major }}' extension/tasks/dependabot/dependabotV1/task.json``" > extension/tasks/dependabot/dependabotV1/task.json` + echo "`jq '.version.Minor=${{ steps.gitversion.outputs.minor }}' extension/tasks/dependabot/dependabotV1/task.json``" > extension/tasks/dependabot/dependabotV1/task.json` + echo "`jq '.version.Patch=${{ github.run_number }}' extension/tasks/dependabot/dependabotV1/task.json``" > extension/tasks/dependabot/dependabotV1/task.json` - name: Create Extension (dev) run: > diff --git a/docs/extension.md b/docs/extension.md index 50af697e..95013b7c 100644 --- a/docs/extension.md +++ b/docs/extension.md @@ -28,22 +28,28 @@ npm install ```bash cd extension -npm run build:prod +npm run build ``` To generate the Azure DevOps `.vsix` extension package for testing, you'll first need to [create a publisher account](https://learn.microsoft.com/en-us/azure/devops/extend/publish/overview?view=azure-devops#create-a-publisher) on the [Visual Studio Marketplace Publishing Portal](https://marketplace.visualstudio.com/manage/createpublisher?managePageRedirect=true). After this, override your publisher ID below and generate the extension with: ```bash -npx tfx-cli extension create --overrides-file overrides.local.json --override "{\"publisher\": \"your-publisher-id-here\"}" --json5 +npm run package -- --overrides-file overrides.local.json --override "{\"publisher\": \"your-publisher-id-here\"}" ``` ## Installing the extension To test the extension in Azure DevOps, you'll first need to build the extension `.vsix` file (see above). After this, [publish your extension](https://learn.microsoft.com/en-us/azure/devops/extend/publish/overview?view=azure-devops#publish-your-extension), then [install your extension](https://learn.microsoft.com/en-us/azure/devops/extend/publish/overview?view=azure-devops#install-your-extension). +## Running the task locally + +```bash +npm start +``` + ## Running the unit tests ```bash cd extension -npm run test +npm test ``` diff --git a/extension/.gitignore b/extension/.gitignore index 57c2ca3a..e949d8ed 100644 --- a/extension/.gitignore +++ b/extension/.gitignore @@ -1,4 +1,4 @@ node_modules .taskkey -task/**/*.js +**/*.js *.vsix \ No newline at end of file diff --git a/extension/overrides.local.json b/extension/overrides.local.json index caa39dcf..4f15562d 100644 --- a/extension/overrides.local.json +++ b/extension/overrides.local.json @@ -1,5 +1,5 @@ { "id": "dependabot-local", - "version": "0.1.0.6", + "version": "0.1.0.0", "name": "Dependabot (Local)" } diff --git a/extension/package.json b/extension/package.json index b4bff593..2ce3de93 100644 --- a/extension/package.json +++ b/extension/package.json @@ -4,8 +4,10 @@ "description": "Dependabot Azure DevOps task", "main": "''", "scripts": { + "postdependencies": "cp -r node_modules tasks/dependabot/dependabotV1/node_modules", "build": "tsc -p .", - "build:prod": "npm run build && cp -r node_modules task/node_modules", + "package": "npx tfx-cli extension create --json5", + "start": "node tasks/dependabot/dependabotV1/index.js", "test": "jest" }, "repository": { diff --git a/extension/task/icon.png b/extension/tasks/dependabot/dependabotV1/icon.png similarity index 100% rename from extension/task/icon.png rename to extension/tasks/dependabot/dependabotV1/icon.png diff --git a/extension/task/index.ts b/extension/tasks/dependabot/dependabotV1/index.ts similarity index 97% rename from extension/task/index.ts rename to extension/tasks/dependabot/dependabotV1/index.ts index b2d5ddfc..e1f7ceb2 100644 --- a/extension/task/index.ts +++ b/extension/tasks/dependabot/dependabotV1/index.ts @@ -1,9 +1,9 @@ import * as tl from 'azure-pipelines-task-lib/task'; import { ToolRunner } from 'azure-pipelines-task-lib/toolrunner'; -import { IDependabotRegistry, IDependabotUpdate } from './IDependabotConfig'; -import getSharedVariables from './utils/getSharedVariables'; -import { parseConfigFile } from './utils/parseConfigFile'; -import { resolveAzureDevOpsIdentities } from './utils/resolveAzureDevOpsIdentities'; +import { IDependabotRegistry, IDependabotUpdate } from '../../utils/IDependabotConfig'; +import getSharedVariables from '../../utils/getSharedVariables'; +import { parseConfigFile } from '../../utils/parseConfigFile'; +import { resolveAzureDevOpsIdentities } from '../../utils/resolveAzureDevOpsIdentities'; async function run() { try { diff --git a/extension/task/task.json b/extension/tasks/dependabot/dependabotV1/task.json similarity index 99% rename from extension/task/task.json rename to extension/tasks/dependabot/dependabotV1/task.json index bdb71f26..b7368afa 100644 --- a/extension/task/task.json +++ b/extension/tasks/dependabot/dependabotV1/task.json @@ -14,7 +14,7 @@ "demands": [], "version": { "Major": 1, - "Minor": 6, + "Minor": 0, "Patch": 0 }, "instanceNameFormat": "Dependabot", diff --git a/extension/task/IDependabotConfig.ts b/extension/tasks/utils/IDependabotConfig.ts similarity index 100% rename from extension/task/IDependabotConfig.ts rename to extension/tasks/utils/IDependabotConfig.ts diff --git a/extension/task/utils/convertPlaceholder.ts b/extension/tasks/utils/convertPlaceholder.ts similarity index 100% rename from extension/task/utils/convertPlaceholder.ts rename to extension/tasks/utils/convertPlaceholder.ts diff --git a/extension/task/utils/extractHostname.ts b/extension/tasks/utils/extractHostname.ts similarity index 100% rename from extension/task/utils/extractHostname.ts rename to extension/tasks/utils/extractHostname.ts diff --git a/extension/task/utils/extractOrganization.ts b/extension/tasks/utils/extractOrganization.ts similarity index 100% rename from extension/task/utils/extractOrganization.ts rename to extension/tasks/utils/extractOrganization.ts diff --git a/extension/task/utils/extractVirtualDirectory.ts b/extension/tasks/utils/extractVirtualDirectory.ts similarity index 100% rename from extension/task/utils/extractVirtualDirectory.ts rename to extension/tasks/utils/extractVirtualDirectory.ts diff --git a/extension/task/utils/getAzureDevOpsAccessToken.ts b/extension/tasks/utils/getAzureDevOpsAccessToken.ts similarity index 100% rename from extension/task/utils/getAzureDevOpsAccessToken.ts rename to extension/tasks/utils/getAzureDevOpsAccessToken.ts diff --git a/extension/task/utils/getDockerImageTag.ts b/extension/tasks/utils/getDockerImageTag.ts similarity index 100% rename from extension/task/utils/getDockerImageTag.ts rename to extension/tasks/utils/getDockerImageTag.ts diff --git a/extension/task/utils/getGithubAccessToken.ts b/extension/tasks/utils/getGithubAccessToken.ts similarity index 100% rename from extension/task/utils/getGithubAccessToken.ts rename to extension/tasks/utils/getGithubAccessToken.ts diff --git a/extension/task/utils/getSharedVariables.ts b/extension/tasks/utils/getSharedVariables.ts similarity index 100% rename from extension/task/utils/getSharedVariables.ts rename to extension/tasks/utils/getSharedVariables.ts diff --git a/extension/task/utils/parseConfigFile.ts b/extension/tasks/utils/parseConfigFile.ts similarity index 99% rename from extension/task/utils/parseConfigFile.ts rename to extension/tasks/utils/parseConfigFile.ts index e158f679..356aa7f5 100644 --- a/extension/task/utils/parseConfigFile.ts +++ b/extension/tasks/utils/parseConfigFile.ts @@ -5,7 +5,7 @@ import * as fs from 'fs'; import { load } from 'js-yaml'; import * as path from 'path'; import { URL } from 'url'; -import { IDependabotConfig, IDependabotRegistry, IDependabotUpdate } from '../IDependabotConfig'; +import { IDependabotConfig, IDependabotRegistry, IDependabotUpdate } from './IDependabotConfig'; import { convertPlaceholder } from './convertPlaceholder'; import { ISharedVariables } from './getSharedVariables'; diff --git a/extension/task/utils/resolveAzureDevOpsIdentities.ts b/extension/tasks/utils/resolveAzureDevOpsIdentities.ts similarity index 100% rename from extension/task/utils/resolveAzureDevOpsIdentities.ts rename to extension/tasks/utils/resolveAzureDevOpsIdentities.ts diff --git a/extension/tests/utils/convertPlaceholder.test.ts b/extension/tests/utils/convertPlaceholder.test.ts index eed88400..f63dfd0b 100644 --- a/extension/tests/utils/convertPlaceholder.test.ts +++ b/extension/tests/utils/convertPlaceholder.test.ts @@ -1,4 +1,4 @@ -import { extractPlaceholder } from '../../task/utils/convertPlaceholder'; +import { extractPlaceholder } from '../../tasks/utils/convertPlaceholder'; describe('Parse property placeholder', () => { it('Should return key with underscores', () => { diff --git a/extension/tests/utils/extractHostname.test.ts b/extension/tests/utils/extractHostname.test.ts index ca8d01f7..00570b41 100644 --- a/extension/tests/utils/extractHostname.test.ts +++ b/extension/tests/utils/extractHostname.test.ts @@ -1,4 +1,4 @@ -import extractHostname from '../../task/utils/extractHostname'; +import extractHostname from '../../tasks/utils/extractHostname'; describe('Extract hostname', () => { it('Should convert old *.visualstudio.com hostname to dev.azure.com', () => { diff --git a/extension/tests/utils/extractOrganization.test.ts b/extension/tests/utils/extractOrganization.test.ts index a827bae0..738201ae 100644 --- a/extension/tests/utils/extractOrganization.test.ts +++ b/extension/tests/utils/extractOrganization.test.ts @@ -1,4 +1,4 @@ -import extractOrganization from '../../task/utils/extractOrganization'; +import extractOrganization from '../../tasks/utils/extractOrganization'; describe('Extract organization name', () => { it('Should extract organization for on-premise domain', () => { diff --git a/extension/tests/utils/extractVirtualDirectory.test.ts b/extension/tests/utils/extractVirtualDirectory.test.ts index 35a734b8..ead119f5 100644 --- a/extension/tests/utils/extractVirtualDirectory.test.ts +++ b/extension/tests/utils/extractVirtualDirectory.test.ts @@ -1,4 +1,4 @@ -import extractVirtualDirectory from '../../task/utils/extractVirtualDirectory'; +import extractVirtualDirectory from '../../tasks/utils/extractVirtualDirectory'; describe('Extract virtual directory', () => { it('Should extract virtual directory', () => { diff --git a/extension/tests/utils/parseConfigFile.test.ts b/extension/tests/utils/parseConfigFile.test.ts index 864218a2..a97a9ffe 100644 --- a/extension/tests/utils/parseConfigFile.test.ts +++ b/extension/tests/utils/parseConfigFile.test.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import { load } from 'js-yaml'; -import { IDependabotRegistry, IDependabotUpdate } from '../../task/IDependabotConfig'; -import { parseRegistries, parseUpdates, validateConfiguration } from '../../task/utils/parseConfigFile'; +import { IDependabotRegistry, IDependabotUpdate } from '../../tasks/utils/IDependabotConfig'; +import { parseRegistries, parseUpdates, validateConfiguration } from '../../tasks/utils/parseConfigFile'; describe('Parse configuration file', () => { it('Parsing works as expected', () => { diff --git a/extension/tests/utils/resolveAzureDevOpsIdentities.test.ts b/extension/tests/utils/resolveAzureDevOpsIdentities.test.ts index d3e54d4c..87953134 100644 --- a/extension/tests/utils/resolveAzureDevOpsIdentities.test.ts +++ b/extension/tests/utils/resolveAzureDevOpsIdentities.test.ts @@ -1,6 +1,6 @@ import axios from 'axios'; import { describe } from 'node:test'; -import { isHostedAzureDevOps, resolveAzureDevOpsIdentities } from '../../task/utils/resolveAzureDevOpsIdentities'; +import { isHostedAzureDevOps, resolveAzureDevOpsIdentities } from '../../tasks/utils/resolveAzureDevOpsIdentities'; describe('isHostedAzureDevOps', () => { it('Old visualstudio url is hosted.', () => { diff --git a/extension/vss-extension.json b/extension/vss-extension.json index 7be9b6ec..c7f1eb70 100644 --- a/extension/vss-extension.json +++ b/extension/vss-extension.json @@ -31,7 +31,7 @@ }, "files": [ { - "path": "task" + "path": "tasks" }, { "path": "images", @@ -44,7 +44,7 @@ "type": "ms.vss-distributed-task.task", "targets": ["ms.vss-distributed-task.tasks"], "properties": { - "name": "task" + "name": "tasks/dependabot" } } ] From a3000cfab3e95dfaf6c7591fe0a61b8681217d3a Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Wed, 4 Sep 2024 10:58:50 +1200 Subject: [PATCH 12/57] Fix typos --- .github/workflows/extension.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/extension.yml b/.github/workflows/extension.yml index b7ca1bfc..8dab185e 100644 --- a/.github/workflows/extension.yml +++ b/.github/workflows/extension.yml @@ -76,9 +76,9 @@ jobs: - name: Update version numbers in task.json run: | - echo "`jq '.version.Major=${{ steps.gitversion.outputs.major }}' extension/tasks/dependabot/dependabotV1/task.json``" > extension/tasks/dependabot/dependabotV1/task.json` - echo "`jq '.version.Minor=${{ steps.gitversion.outputs.minor }}' extension/tasks/dependabot/dependabotV1/task.json``" > extension/tasks/dependabot/dependabotV1/task.json` - echo "`jq '.version.Patch=${{ github.run_number }}' extension/tasks/dependabot/dependabotV1/task.json``" > extension/tasks/dependabot/dependabotV1/task.json` + echo "`jq '.version.Major=${{ steps.gitversion.outputs.major }}' extension/tasks/dependabot/dependabotV1/task.json`" > extension/tasks/dependabot/dependabotV1/task.json + echo "`jq '.version.Minor=${{ steps.gitversion.outputs.minor }}' extension/tasks/dependabot/dependabotV1/task.json`" > extension/tasks/dependabot/dependabotV1/task.json + echo "`jq '.version.Patch=${{ github.run_number }}' extension/tasks/dependabot/dependabotV1/task.json`" > extension/tasks/dependabot/dependabotV1/task.json - name: Create Extension (dev) run: > From 49b851a38761e000a0507e511c706c1ec80db170 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Wed, 4 Sep 2024 12:24:07 +1200 Subject: [PATCH 13/57] Fix build --- docs/extension.md | 2 +- extension/overrides.local.json | 2 +- extension/tasks/dependabot/dependabotV1/index.ts | 8 ++++---- .../dependabotV1}/utils/IDependabotConfig.ts | 0 .../dependabotV1}/utils/convertPlaceholder.ts | 0 .../dependabotV1}/utils/extractHostname.ts | 0 .../dependabotV1}/utils/extractOrganization.ts | 0 .../dependabotV1}/utils/extractVirtualDirectory.ts | 0 .../dependabotV1}/utils/getAzureDevOpsAccessToken.ts | 0 .../dependabotV1}/utils/getDockerImageTag.ts | 0 .../dependabotV1}/utils/getGithubAccessToken.ts | 0 .../dependabotV1}/utils/getSharedVariables.ts | 0 .../dependabotV1}/utils/parseConfigFile.ts | 0 .../dependabotV1}/utils/resolveAzureDevOpsIdentities.ts | 0 extension/vss-extension.json | 2 +- 15 files changed, 7 insertions(+), 7 deletions(-) rename extension/tasks/{ => dependabot/dependabotV1}/utils/IDependabotConfig.ts (100%) rename extension/tasks/{ => dependabot/dependabotV1}/utils/convertPlaceholder.ts (100%) rename extension/tasks/{ => dependabot/dependabotV1}/utils/extractHostname.ts (100%) rename extension/tasks/{ => dependabot/dependabotV1}/utils/extractOrganization.ts (100%) rename extension/tasks/{ => dependabot/dependabotV1}/utils/extractVirtualDirectory.ts (100%) rename extension/tasks/{ => dependabot/dependabotV1}/utils/getAzureDevOpsAccessToken.ts (100%) rename extension/tasks/{ => dependabot/dependabotV1}/utils/getDockerImageTag.ts (100%) rename extension/tasks/{ => dependabot/dependabotV1}/utils/getGithubAccessToken.ts (100%) rename extension/tasks/{ => dependabot/dependabotV1}/utils/getSharedVariables.ts (100%) rename extension/tasks/{ => dependabot/dependabotV1}/utils/parseConfigFile.ts (100%) rename extension/tasks/{ => dependabot/dependabotV1}/utils/resolveAzureDevOpsIdentities.ts (100%) diff --git a/docs/extension.md b/docs/extension.md index 95013b7c..408f0755 100644 --- a/docs/extension.md +++ b/docs/extension.md @@ -34,7 +34,7 @@ npm run build To generate the Azure DevOps `.vsix` extension package for testing, you'll first need to [create a publisher account](https://learn.microsoft.com/en-us/azure/devops/extend/publish/overview?view=azure-devops#create-a-publisher) on the [Visual Studio Marketplace Publishing Portal](https://marketplace.visualstudio.com/manage/createpublisher?managePageRedirect=true). After this, override your publisher ID below and generate the extension with: ```bash -npm run package -- --overrides-file overrides.local.json --override "{\"publisher\": \"your-publisher-id-here\"}" +npm run package -- --overrides-file overrides.local.json --rev-version --publisher your-publisher-id-here ``` ## Installing the extension diff --git a/extension/overrides.local.json b/extension/overrides.local.json index 4f15562d..8063d93a 100644 --- a/extension/overrides.local.json +++ b/extension/overrides.local.json @@ -1,5 +1,5 @@ { "id": "dependabot-local", - "version": "0.1.0.0", + "version": "1.0.0.0", "name": "Dependabot (Local)" } diff --git a/extension/tasks/dependabot/dependabotV1/index.ts b/extension/tasks/dependabot/dependabotV1/index.ts index e1f7ceb2..774f1db9 100644 --- a/extension/tasks/dependabot/dependabotV1/index.ts +++ b/extension/tasks/dependabot/dependabotV1/index.ts @@ -1,9 +1,9 @@ import * as tl from 'azure-pipelines-task-lib/task'; import { ToolRunner } from 'azure-pipelines-task-lib/toolrunner'; -import { IDependabotRegistry, IDependabotUpdate } from '../../utils/IDependabotConfig'; -import getSharedVariables from '../../utils/getSharedVariables'; -import { parseConfigFile } from '../../utils/parseConfigFile'; -import { resolveAzureDevOpsIdentities } from '../../utils/resolveAzureDevOpsIdentities'; +import { IDependabotRegistry, IDependabotUpdate } from './utils/IDependabotConfig'; +import getSharedVariables from './utils/getSharedVariables'; +import { parseConfigFile } from './utils/parseConfigFile'; +import { resolveAzureDevOpsIdentities } from './utils/resolveAzureDevOpsIdentities'; async function run() { try { diff --git a/extension/tasks/utils/IDependabotConfig.ts b/extension/tasks/dependabot/dependabotV1/utils/IDependabotConfig.ts similarity index 100% rename from extension/tasks/utils/IDependabotConfig.ts rename to extension/tasks/dependabot/dependabotV1/utils/IDependabotConfig.ts diff --git a/extension/tasks/utils/convertPlaceholder.ts b/extension/tasks/dependabot/dependabotV1/utils/convertPlaceholder.ts similarity index 100% rename from extension/tasks/utils/convertPlaceholder.ts rename to extension/tasks/dependabot/dependabotV1/utils/convertPlaceholder.ts diff --git a/extension/tasks/utils/extractHostname.ts b/extension/tasks/dependabot/dependabotV1/utils/extractHostname.ts similarity index 100% rename from extension/tasks/utils/extractHostname.ts rename to extension/tasks/dependabot/dependabotV1/utils/extractHostname.ts diff --git a/extension/tasks/utils/extractOrganization.ts b/extension/tasks/dependabot/dependabotV1/utils/extractOrganization.ts similarity index 100% rename from extension/tasks/utils/extractOrganization.ts rename to extension/tasks/dependabot/dependabotV1/utils/extractOrganization.ts diff --git a/extension/tasks/utils/extractVirtualDirectory.ts b/extension/tasks/dependabot/dependabotV1/utils/extractVirtualDirectory.ts similarity index 100% rename from extension/tasks/utils/extractVirtualDirectory.ts rename to extension/tasks/dependabot/dependabotV1/utils/extractVirtualDirectory.ts diff --git a/extension/tasks/utils/getAzureDevOpsAccessToken.ts b/extension/tasks/dependabot/dependabotV1/utils/getAzureDevOpsAccessToken.ts similarity index 100% rename from extension/tasks/utils/getAzureDevOpsAccessToken.ts rename to extension/tasks/dependabot/dependabotV1/utils/getAzureDevOpsAccessToken.ts diff --git a/extension/tasks/utils/getDockerImageTag.ts b/extension/tasks/dependabot/dependabotV1/utils/getDockerImageTag.ts similarity index 100% rename from extension/tasks/utils/getDockerImageTag.ts rename to extension/tasks/dependabot/dependabotV1/utils/getDockerImageTag.ts diff --git a/extension/tasks/utils/getGithubAccessToken.ts b/extension/tasks/dependabot/dependabotV1/utils/getGithubAccessToken.ts similarity index 100% rename from extension/tasks/utils/getGithubAccessToken.ts rename to extension/tasks/dependabot/dependabotV1/utils/getGithubAccessToken.ts diff --git a/extension/tasks/utils/getSharedVariables.ts b/extension/tasks/dependabot/dependabotV1/utils/getSharedVariables.ts similarity index 100% rename from extension/tasks/utils/getSharedVariables.ts rename to extension/tasks/dependabot/dependabotV1/utils/getSharedVariables.ts diff --git a/extension/tasks/utils/parseConfigFile.ts b/extension/tasks/dependabot/dependabotV1/utils/parseConfigFile.ts similarity index 100% rename from extension/tasks/utils/parseConfigFile.ts rename to extension/tasks/dependabot/dependabotV1/utils/parseConfigFile.ts diff --git a/extension/tasks/utils/resolveAzureDevOpsIdentities.ts b/extension/tasks/dependabot/dependabotV1/utils/resolveAzureDevOpsIdentities.ts similarity index 100% rename from extension/tasks/utils/resolveAzureDevOpsIdentities.ts rename to extension/tasks/dependabot/dependabotV1/utils/resolveAzureDevOpsIdentities.ts diff --git a/extension/vss-extension.json b/extension/vss-extension.json index c7f1eb70..7fbf213f 100644 --- a/extension/vss-extension.json +++ b/extension/vss-extension.json @@ -2,7 +2,7 @@ "manifestVersion": 1, "id": "dependabot", "name": "Dependabot", - "version": "0.1.0", + "version": "1.0.0", "publisher": "tingle-software", "public": false, "targets": [ From 8c456d3d72aa5ba4f61f06d4df8e3045638ac0e7 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Wed, 4 Sep 2024 12:43:54 +1200 Subject: [PATCH 14/57] Fix build --- extension/jest.config.ts | 3 +-- .../dependabotV1}/utils/convertPlaceholder.test.ts | 2 +- .../dependabotV1}/utils/extractHostname.test.ts | 2 +- .../dependabotV1}/utils/extractOrganization.test.ts | 2 +- .../dependabotV1}/utils/extractVirtualDirectory.test.ts | 2 +- .../dependabotV1}/utils/parseConfigFile.test.ts | 8 ++++---- .../utils/resolveAzureDevOpsIdentities.test.ts | 2 +- extension/tests/{utils => config}/dependabot.yml | 0 extension/tests/{utils => config}/sample-registries.yml | 0 9 files changed, 10 insertions(+), 11 deletions(-) rename extension/{tests => tasks/dependabot/dependabotV1}/utils/convertPlaceholder.test.ts (93%) rename extension/{tests => tasks/dependabot/dependabotV1}/utils/extractHostname.test.ts (91%) rename extension/{tests => tasks/dependabot/dependabotV1}/utils/extractOrganization.test.ts (90%) rename extension/{tests => tasks/dependabot/dependabotV1}/utils/extractVirtualDirectory.test.ts (86%) rename extension/{tests => tasks/dependabot/dependabotV1}/utils/parseConfigFile.test.ts (97%) rename extension/{tests => tasks/dependabot/dependabotV1}/utils/resolveAzureDevOpsIdentities.test.ts (98%) rename extension/tests/{utils => config}/dependabot.yml (100%) rename extension/tests/{utils => config}/sample-registries.yml (100%) diff --git a/extension/jest.config.ts b/extension/jest.config.ts index e095ec23..e82e036a 100644 --- a/extension/jest.config.ts +++ b/extension/jest.config.ts @@ -7,8 +7,7 @@ const config: Config.InitialOptions = { // "^.+\\.test.tsx?$": "ts-jest", // }, testEnvironment: 'node', - preset: 'ts-jest', - rootDir: './tests', + preset: 'ts-jest' }; export default config; diff --git a/extension/tests/utils/convertPlaceholder.test.ts b/extension/tasks/dependabot/dependabotV1/utils/convertPlaceholder.test.ts similarity index 93% rename from extension/tests/utils/convertPlaceholder.test.ts rename to extension/tasks/dependabot/dependabotV1/utils/convertPlaceholder.test.ts index f63dfd0b..d16418a6 100644 --- a/extension/tests/utils/convertPlaceholder.test.ts +++ b/extension/tasks/dependabot/dependabotV1/utils/convertPlaceholder.test.ts @@ -1,4 +1,4 @@ -import { extractPlaceholder } from '../../tasks/utils/convertPlaceholder'; +import { extractPlaceholder } from './convertPlaceholder'; describe('Parse property placeholder', () => { it('Should return key with underscores', () => { diff --git a/extension/tests/utils/extractHostname.test.ts b/extension/tasks/dependabot/dependabotV1/utils/extractHostname.test.ts similarity index 91% rename from extension/tests/utils/extractHostname.test.ts rename to extension/tasks/dependabot/dependabotV1/utils/extractHostname.test.ts index 00570b41..78044219 100644 --- a/extension/tests/utils/extractHostname.test.ts +++ b/extension/tasks/dependabot/dependabotV1/utils/extractHostname.test.ts @@ -1,4 +1,4 @@ -import extractHostname from '../../tasks/utils/extractHostname'; +import extractHostname from './extractHostname'; describe('Extract hostname', () => { it('Should convert old *.visualstudio.com hostname to dev.azure.com', () => { diff --git a/extension/tests/utils/extractOrganization.test.ts b/extension/tasks/dependabot/dependabotV1/utils/extractOrganization.test.ts similarity index 90% rename from extension/tests/utils/extractOrganization.test.ts rename to extension/tasks/dependabot/dependabotV1/utils/extractOrganization.test.ts index 738201ae..ebdb8209 100644 --- a/extension/tests/utils/extractOrganization.test.ts +++ b/extension/tasks/dependabot/dependabotV1/utils/extractOrganization.test.ts @@ -1,4 +1,4 @@ -import extractOrganization from '../../tasks/utils/extractOrganization'; +import extractOrganization from './extractOrganization'; describe('Extract organization name', () => { it('Should extract organization for on-premise domain', () => { diff --git a/extension/tests/utils/extractVirtualDirectory.test.ts b/extension/tasks/dependabot/dependabotV1/utils/extractVirtualDirectory.test.ts similarity index 86% rename from extension/tests/utils/extractVirtualDirectory.test.ts rename to extension/tasks/dependabot/dependabotV1/utils/extractVirtualDirectory.test.ts index ead119f5..71ce773d 100644 --- a/extension/tests/utils/extractVirtualDirectory.test.ts +++ b/extension/tasks/dependabot/dependabotV1/utils/extractVirtualDirectory.test.ts @@ -1,4 +1,4 @@ -import extractVirtualDirectory from '../../tasks/utils/extractVirtualDirectory'; +import extractVirtualDirectory from './extractVirtualDirectory'; describe('Extract virtual directory', () => { it('Should extract virtual directory', () => { diff --git a/extension/tests/utils/parseConfigFile.test.ts b/extension/tasks/dependabot/dependabotV1/utils/parseConfigFile.test.ts similarity index 97% rename from extension/tests/utils/parseConfigFile.test.ts rename to extension/tasks/dependabot/dependabotV1/utils/parseConfigFile.test.ts index a97a9ffe..2e908379 100644 --- a/extension/tests/utils/parseConfigFile.test.ts +++ b/extension/tasks/dependabot/dependabotV1/utils/parseConfigFile.test.ts @@ -1,11 +1,11 @@ import * as fs from 'fs'; import { load } from 'js-yaml'; -import { IDependabotRegistry, IDependabotUpdate } from '../../tasks/utils/IDependabotConfig'; -import { parseRegistries, parseUpdates, validateConfiguration } from '../../tasks/utils/parseConfigFile'; +import { IDependabotRegistry, IDependabotUpdate } from './IDependabotConfig'; +import { parseRegistries, parseUpdates, validateConfiguration } from './parseConfigFile'; describe('Parse configuration file', () => { it('Parsing works as expected', () => { - let config: any = load(fs.readFileSync('tests/utils/dependabot.yml', 'utf-8')); + let config: any = load(fs.readFileSync('tests/config/dependabot.yml', 'utf-8')); let updates = parseUpdates(config); expect(updates.length).toBe(3); @@ -36,7 +36,7 @@ describe('Parse configuration file', () => { describe('Parse registries', () => { it('Parsing works as expected', () => { - let config: any = load(fs.readFileSync('tests/utils/sample-registries.yml', 'utf-8')); + let config: any = load(fs.readFileSync('tests/config/sample-registries.yml', 'utf-8')); let registries = parseRegistries(config); expect(Object.keys(registries).length).toBe(11); diff --git a/extension/tests/utils/resolveAzureDevOpsIdentities.test.ts b/extension/tasks/dependabot/dependabotV1/utils/resolveAzureDevOpsIdentities.test.ts similarity index 98% rename from extension/tests/utils/resolveAzureDevOpsIdentities.test.ts rename to extension/tasks/dependabot/dependabotV1/utils/resolveAzureDevOpsIdentities.test.ts index 87953134..1e0d91cf 100644 --- a/extension/tests/utils/resolveAzureDevOpsIdentities.test.ts +++ b/extension/tasks/dependabot/dependabotV1/utils/resolveAzureDevOpsIdentities.test.ts @@ -1,6 +1,6 @@ import axios from 'axios'; import { describe } from 'node:test'; -import { isHostedAzureDevOps, resolveAzureDevOpsIdentities } from '../../tasks/utils/resolveAzureDevOpsIdentities'; +import { isHostedAzureDevOps, resolveAzureDevOpsIdentities } from './resolveAzureDevOpsIdentities'; describe('isHostedAzureDevOps', () => { it('Old visualstudio url is hosted.', () => { diff --git a/extension/tests/utils/dependabot.yml b/extension/tests/config/dependabot.yml similarity index 100% rename from extension/tests/utils/dependabot.yml rename to extension/tests/config/dependabot.yml diff --git a/extension/tests/utils/sample-registries.yml b/extension/tests/config/sample-registries.yml similarity index 100% rename from extension/tests/utils/sample-registries.yml rename to extension/tests/config/sample-registries.yml From 189ee76182e5328ec85688d9434210feebeb6cf8 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Wed, 4 Sep 2024 23:17:01 +1200 Subject: [PATCH 15/57] Fix merge issues --- extension/tasks/dependabot/dependabotV1/task.json | 2 +- .../dependabotV1/utils/getSharedVariables.ts | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV1/task.json b/extension/tasks/dependabot/dependabotV1/task.json index e608c69f..f062ac56 100644 --- a/extension/tasks/dependabot/dependabotV1/task.json +++ b/extension/tasks/dependabot/dependabotV1/task.json @@ -18,7 +18,7 @@ "Patch": 0 }, "deprecated": true, - "deprecationMessage": "This task version is deprecated due to updates in dependabot-core. The recommended approach to running dependabot has changed, and this task is no longer maintained. Please upgrade to the latest version to continue receiving fixes and features. More details: https://github.com/tinglesoftware/dependabot-azure-devops/discussions/1317.", + "deprecationMessage": "The recommended approach to running dependabot has changed since the creation of this task, and is no longer maintained. Please upgrade to the latest version to continue receiving fixes and features. More details: https://github.com/tinglesoftware/dependabot-azure-devops/discussions/1317.", "instanceNameFormat": "Dependabot", "minimumAgentVersion": "3.232.1", "groups": [ diff --git a/extension/tasks/dependabot/dependabotV1/utils/getSharedVariables.ts b/extension/tasks/dependabot/dependabotV1/utils/getSharedVariables.ts index 34100a26..87a90ab6 100644 --- a/extension/tasks/dependabot/dependabotV1/utils/getSharedVariables.ts +++ b/extension/tasks/dependabot/dependabotV1/utils/getSharedVariables.ts @@ -27,9 +27,6 @@ export interface ISharedVariables { /** Whether the repository was overridden via input */ repositoryOverridden: boolean; - /** Organisation API endpoint URL */ - apiEndpointUrl: string; - /** The github token */ githubAccessToken: string; /** The access User for Azure DevOps Repos */ @@ -89,9 +86,9 @@ export interface ISharedVariables { */ export default function getSharedVariables(): ISharedVariables { let organizationUrl = tl.getVariable('System.TeamFoundationCollectionUri'); - //convert url string into a valid JS URL object let formattedOrganizationUrl = new URL(organizationUrl); + let protocol: string = formattedOrganizationUrl.protocol.slice(0, -1); let hostname: string = extractHostname(formattedOrganizationUrl); let port: string = formattedOrganizationUrl.port; @@ -106,9 +103,6 @@ export default function getSharedVariables(): ISharedVariables { } repository = encodeURI(repository); // encode special characters like spaces - const virtualDirectorySuffix = virtualDirectory?.length > 0 ? `${virtualDirectory}/` : ''; - let apiEndpointUrl = `${protocol}://${hostname}:${port}/${virtualDirectorySuffix}`; - // Prepare the access credentials let githubAccessToken: string = getGithubAccessToken(); let systemAccessUser: string = tl.getInput('azureDevOpsUser'); @@ -159,8 +153,6 @@ export default function getSharedVariables(): ISharedVariables { repository, repositoryOverridden, - apiEndpointUrl, - githubAccessToken, systemAccessUser, systemAccessToken, From 47a592ac807823bc8bf3a788d6f17e23118b8fdc Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Fri, 6 Sep 2024 01:23:30 +1200 Subject: [PATCH 16/57] Remove unused task inputs --- .../tasks/dependabot/dependabotV2/task.json | 49 ------------------- 1 file changed, 49 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/task.json b/extension/tasks/dependabot/dependabotV2/task.json index ef09db0e..5096b307 100644 --- a/extension/tasks/dependabot/dependabotV2/task.json +++ b/extension/tasks/dependabot/dependabotV2/task.json @@ -48,26 +48,6 @@ } ], "inputs": [ - { - "name": "useUpdateScriptvNext", - "type": "boolean", - "groupName": "advanced", - "label": "Use latest update script (vNext) (Experimental)", - "defaultValue": "false", - "required": false, - "helpMarkDown": "Determines if the task will use the newest 'vNext' update script instead of the default update script. This Defaults to `false`. See the [vNext update script documentation](https://github.com/tinglesoftware/dependabot-azure-devops/pull/1186) for more information." - }, - - { - "name": "failOnException", - "type": "boolean", - "groupName": "advanced", - "label": "Fail task when an update exception occurs.", - "defaultValue": true, - "required": false, - "helpMarkDown": "When set to `true`, a failure in updating a single dependency will cause the container execution to fail thereby causing the task to fail. This is important when you want a single failure to prevent trying to update other dependencies." - }, - { "name": "skipPullRequests", "type": "boolean", @@ -149,7 +129,6 @@ "helpMarkDown": "A personal access token of the user of that shall be used to approve the created PR automatically. If the same user that creates the PR should approve, this can be left empty. This won't work with if the Build Service with the build service account!", "visibleRule": "autoApprove=true" }, - { "name": "gitHubConnection", "type": "connectedService:github:OAuth,PersonalAccessToken,InstallationToken,Token", @@ -168,7 +147,6 @@ "required": false, "helpMarkDown": "The raw Personal Access Token for accessing GitHub repositories. Use this in place of `gitHubConnection` such as when it is not possible to create a service connection." }, - { "name": "securityAdvisoriesFile", "type": "string", @@ -177,7 +155,6 @@ "helpMarkDown": "The file containing security advisories.", "required": false }, - { "name": "azureDevOpsServiceConnection", "type": "connectedService:Externaltfs", @@ -226,32 +203,6 @@ "label": "Space-separated list of dependency updates requirements to be excluded.", "required": false, "helpMarkDown": "Exclude certain dependency updates requirements. See list of allowed values [here](https://github.com/dependabot/dependabot-core/issues/600#issuecomment-407808103). Useful if you have lots of dependencies and the update script too slow. The values provided are space-separated. Example: `own all` to only use the `none` version requirement." - }, - { - "name": "dockerImageTag", - "type": "string", - "groupName": "advanced", - "label": "Tag of the docker image to be pulled.", - "required": false, - "helpMarkDown": "The image tag to use when pulling the docker container used by the task. A tag also defines the version. By default, the task decides which tag/version to use. This can be the latest or most stable version. You can also use `major.minor` format to get the latest patch" - }, - { - "name": "extraEnvironmentVariables", - "type": "string", - "groupName": "advanced", - "label": "Semicolon delimited list of environment variables", - "required": false, - "defaultValue": "", - "helpMarkDown": "A semicolon (`;`) delimited list of environment variables that are sent to the docker container. See possible use case [here](https://github.com/tinglesoftware/dependabot-azure-devops/issues/138)" - }, - { - "name": "forwardHostSshSocket", - "type": "boolean", - "groupName": "advanced", - "label": "Forward the host ssh socket", - "defaultValue": "false", - "required": false, - "helpMarkDown": "Ensure that the host ssh socket is forwarded to the container to authenticate with ssh" } ], "dataSourceBindings": [], From 431a9810a31ab97a5abc5bc48361f8ead8396d24 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Fri, 6 Sep 2024 01:29:10 +1200 Subject: [PATCH 17/57] Fix for 'convertPlaceholder' not accepted built-in DevOps variable names containing '.' --- .../tasks/dependabot/dependabotV2/utils/convertPlaceholder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/tasks/dependabot/dependabotV2/utils/convertPlaceholder.ts b/extension/tasks/dependabot/dependabotV2/utils/convertPlaceholder.ts index aabca849..d4e96d5d 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/convertPlaceholder.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/convertPlaceholder.ts @@ -13,7 +13,7 @@ function convertPlaceholder(input: string): string { } function extractPlaceholder(input: string) { - const regexp: RegExp = new RegExp('\\${{\\s*([a-zA-Z_]+[a-zA-Z0-9_-]*)\\s*}}', 'g'); + const regexp: RegExp = new RegExp('\\${{\\s*([a-zA-Z_]+[a-zA-Z0-9\\._-]*)\\s*}}', 'g'); return matchAll(input, regexp); } From 0aae4770379826863a86bcf65aa46d1f6548395d Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Fri, 6 Sep 2024 01:41:06 +1200 Subject: [PATCH 18/57] Implement closing pull requests --- .../tasks/dependabot/dependabotV2/index.ts | 69 ++++++--- .../azure-devops/AzureDevOpsWebApiClient.ts | 76 +++++++++- .../utils/dependabot-cli/DependabotCli.ts | 67 ++++----- .../dependabot-cli/DependabotJobBuilder.ts | 19 ++- .../DependabotOutputProcessor.ts | 137 ++++++++++++------ .../interfaces/IDependabotUpdateOutput.ts | 2 +- 6 files changed, 267 insertions(+), 103 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index 55dce653..5f84140a 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -8,7 +8,7 @@ import { parseConfigFile } from './utils/parseConfigFile'; import getSharedVariables from './utils/getSharedVariables'; async function run() { - let taskWasSuccessful: boolean = true; + let taskSucceeded: boolean = true; let dependabot: DependabotCli = undefined; try { @@ -18,29 +18,40 @@ async function run() { debug('Checking for `go` install...'); which('go', true); - // Parse the dependabot.yaml configuration file + // Parse task input configuration const taskVariables = getSharedVariables(); + if (!taskVariables) { + throw new Error('Failed to parse task input configuration'); + } + + // Parse dependabot.yaml configuration file const dependabotConfig = await parseConfigFile(taskVariables); + if (!dependabotConfig) { + throw new Error('Failed to parse dependabot.yaml configuration file from the target repository'); + } // Initialise the DevOps API client const azdoApi = new AzureDevOpsWebApiClient( taskVariables.organizationUrl.toString(), taskVariables.systemAccessToken ); + // Fetch the active pull requests created by our user + const myActivePullRequests = await azdoApi.getMyActivePullRequestProperties( + taskVariables.project, taskVariables.repository + ); + // Initialise the Dependabot updater dependabot = new DependabotCli( DependabotCli.CLI_IMAGE_LATEST, // TODO: Add config for this? - new DependabotOutputProcessor(azdoApi, taskVariables), + new DependabotOutputProcessor(azdoApi, myActivePullRequests, taskVariables), taskVariables.debug ); - // Fetch all active Dependabot pull requests from DevOps - const myActivePullRequests = await azdoApi.getMyActivePullRequestProperties( - taskVariables.project, taskVariables.repository - ); - - // Loop through each package ecyosystem and perform updates + // Loop through each 'update' block in dependabot.yaml and perform updates dependabotConfig.updates.forEach(async (update) => { + + // Parse the Dependabot metadata for the existing pull requests that are related to this update + // Dependabot will use this to determine if we need to create new pull requests or update/close existing ones const existingPullRequests = myActivePullRequests .filter(pr => { return pr.properties.find(p => p.name === DependabotOutputProcessor.PR_PROPERTY_NAME_PACKAGE_MANAGER && p.value === update.packageEcosystem); @@ -51,34 +62,50 @@ async function run() { ) }); - // Update all dependencies, this will update create new pull requests - const allDependenciesJob = DependabotJobBuilder.updateAllDependenciesJob(taskVariables, update, dependabotConfig.registries, existingPullRequests); - if ((await dependabot.update(allDependenciesJob)).filter(u => !u.success).length > 0) { - taskWasSuccessful = false; + // Run an update job for "all dependencies"; this will create new pull requests for dependencies that need updating + const allDependenciesJob = DependabotJobBuilder.newUpdateAllJob(taskVariables, update, dependabotConfig.registries, existingPullRequests); + const allDependenciesUpdateOutputs = await dependabot.update(allDependenciesJob); + if (!allDependenciesUpdateOutputs || allDependenciesUpdateOutputs.filter(u => !u.success).length > 0) { + allDependenciesUpdateOutputs.filter(u => !u.success).forEach(u => exception(u.error)); + taskSucceeded = false; } - // Update existing pull requests, this will either resolve merge conflicts or close pull requests that are no longer needed - for (const pr of existingPullRequests) { - const updatePullRequestJob = DependabotJobBuilder.updatePullRequestJob(taskVariables, update, dependabotConfig.registries, existingPullRequests, pr); - if ((await dependabot.update(updatePullRequestJob)).filter(u => !u.success).length > 0) { - taskWasSuccessful = false; + // Run an update job for each existing pull request; this will resolve merge conflicts and close pull requests that are no longer needed + if (!taskVariables.skipPullRequests) { + for (const pr of existingPullRequests) { + const updatePullRequestJob = DependabotJobBuilder.newUpdatePullRequestJob(taskVariables, update, dependabotConfig.registries, existingPullRequests, pr); + const updatePullRequestOutputs = await dependabot.update(updatePullRequestJob); + if (!updatePullRequestOutputs || updatePullRequestOutputs.filter(u => !u.success).length > 0) { + updatePullRequestOutputs.filter(u => !u.success).forEach(u => exception(u.error)); + taskSucceeded = false; + } } + } else if (existingPullRequests.length > 0) { + warning(`Skipping update of existing pull requests as 'skipPullRequests' is set to 'true'`); + return; } + }); setResult( - taskWasSuccessful ? TaskResult.Succeeded : TaskResult.Failed, - taskWasSuccessful ? 'All update jobs completed successfully' : 'One or more update jobs failed, check logs for more information' + taskSucceeded ? TaskResult.Succeeded : TaskResult.Failed, + taskSucceeded ? 'All update jobs completed successfully' : 'One or more update jobs failed, check logs for more information' ); } catch (e) { - error(`Unhandled task exception: ${e}`); setResult(TaskResult.Failed, e?.message); + exception(e); } finally { // TODO: dependabotCli?.cleanup(); } } +function exception(e: Error) { + if (e?.stack) { + error(e.stack); + } +} + run(); diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts index 32b5f31e..a6137959 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts @@ -1,6 +1,6 @@ import { debug, warning, error } from "azure-pipelines-task-lib/task" import { WebApi, getPersonalAccessTokenHandler } from "azure-devops-node-api"; -import { ItemContentType, PullRequestStatus } from "azure-devops-node-api/interfaces/GitInterfaces"; +import { CommentThreadStatus, CommentType, ItemContentType, PullRequestStatus } from "azure-devops-node-api/interfaces/GitInterfaces"; import { IPullRequestProperties } from "./interfaces/IPullRequestProperties"; import { IPullRequest } from "./interfaces/IPullRequest"; @@ -173,6 +173,80 @@ export class AzureDevOpsWebApiClient { } } + // Close a pull request + public async closePullRequest(options: { + project: string, + repository: string, + pullRequestId: number, + comment: string, + deleteSourceBranch: boolean + }): Promise { + console.info(`Closing pull request #${options.pullRequestId}...`); + try { + const userId = await this.getUserId(); + const git = await this.connection.getGitApi(); + + // Add a comment to the pull request, if supplied + if (options.comment) { + console.info(` - Adding comment to pull request...`); + await git.createThread( + { + status: CommentThreadStatus.Closed, + comments: [ + { + author: { + id: userId + }, + content: options.comment, + commentType: CommentType.System + } + ] + }, + options.repository, + options.pullRequestId, + options.project + ); + } + + // Close the pull request + console.info(` - Abandoning pull request...`); + const pullRequest = await git.updatePullRequest( + { + status: PullRequestStatus.Abandoned, + closedBy: { + id: userId + } + }, + options.repository, + options.pullRequestId, + options.project + ); + + // Delete the source branch if required + if (options.deleteSourceBranch) { + console.info(` - Deleting source branch...`); + await git.updateRef( + { + name: `refs/heads/${pullRequest.sourceRefName}`, + oldObjectId: pullRequest.lastMergeSourceCommit.commitId, + newObjectId: "0000000000000000000000000000000000000000", + isLocked: false + }, + options.repository, + '', + options.project + ); + } + + console.info(` - Pull request #${options.pullRequestId} was closed successfully.`); + return true; + } + catch (e) { + error(`Failed to close pull request: ${e}`); + return false; + } + } + private async getUserId(): Promise { return (this.userId ||= (await this.connection.connect()).authenticatedUser?.id || ""); } diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts index 5daebe29..02140a4f 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts @@ -34,7 +34,7 @@ export class DependabotCli { proxyImage?: string, updaterImage?: string } - ): Promise { + ): Promise { // Install dependabot if not already installed await this.ensureToolsAreInstalled(); @@ -69,48 +69,53 @@ export class DependabotCli { writeJobFile(jobInputPath, config); // Run dependabot update - if (!fs.existsSync(jobOutputPath)) { - console.info("Running dependabot update..."); + if (!fs.existsSync(jobOutputPath) || fs.statSync(jobOutputPath)?.size == 0) { + console.info(`Running dependabot update job '${jobInputPath}'...`); const dependabotTool = tool(which("dependabot", true)).arg(dependabotArguments); const dependabotResultCode = await dependabotTool.execAsync({ - silent: !this.debug + silent: !this.debug, + failOnStdErr: false, + ignoreReturnCode: true }); if (dependabotResultCode != 0) { - throw new Error(`Dependabot update failed with exit code ${dependabotResultCode}`); + error(`Dependabot failed with exit code ${dependabotResultCode}`); } } // Process the job output const processedOutputs = Array(); if (fs.existsSync(jobOutputPath)) { - console.info("Processing dependabot update outputs..."); - for (const output of readScenarioOutputs(jobOutputPath)) { - // Documentation on the scenario model can be found here: - // https://github.com/dependabot/cli/blob/main/internal/model/scenario.go - const type = output['type']; - const data = output['expect']?.['data']; - var processedOutput = { - success: true, - error: null, - output: { - type: type, - data: data + const outputs = readScenarioOutputs(jobOutputPath); + if (outputs?.length > 0) { + console.info("Processing Dependabot outputs..."); + for (const output of outputs) { + // Documentation on the scenario model can be found here: + // https://github.com/dependabot/cli/blob/main/internal/model/scenario.go + const type = output['type']; + const data = output['expect']?.['data']; + var processedOutput = { + success: true, + error: null, + output: { + type: type, + data: data + } + }; + try { + processedOutput.success = await this.outputProcessor.process(config, type, data); + } + catch (e) { + processedOutput.success = false; + processedOutput.error = e; + } + finally { + processedOutputs.push(processedOutput); } - }; - try { - processedOutput.success = await this.outputProcessor.process(config, type, data); - } - catch (e) { - processedOutput.success = false; - processedOutput.error = e; - } - finally { - processedOutputs.push(processedOutput); } } } - return processedOutputs; + return processedOutputs.length > 0 ? processedOutputs : undefined; } // Install dependabot if not already installed @@ -159,13 +164,9 @@ function writeJobFile(path: string, config: IDependabotUpdateJob): void { // Documentation on the scenario model can be found here: // https://github.com/dependabot/cli/blob/main/internal/model/scenario.go function readScenarioOutputs(path: string): any[] { - if (!path) { - throw new Error("Scenario file path is required"); - } - const scenarioContent = fs.readFileSync(path, 'utf-8'); if (!scenarioContent || typeof scenarioContent !== 'string') { - throw new Error(`Scenario file could not be read at '${path}'`); + return []; // No outputs or failed scenario } const scenario: any = yaml.load(scenarioContent); diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts index 4c251496..8ee4743d 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts @@ -6,7 +6,7 @@ import { IDependabotUpdateJob } from "./interfaces/IDependabotUpdateJob"; export class DependabotJobBuilder { // Create a dependabot update job that updates all dependencies for a package ecyosystem - public static updateAllDependenciesJob( + public static newUpdateAllJob( variables: ISharedVariables, update: IDependabotUpdate, registries: Record, @@ -24,7 +24,7 @@ export class DependabotJobBuilder { 'allowed-updates': [ { 'update-type': 'all' } // TODO: update.allow ], - 'ignore-conditions': [], // TODO: update.ignore + 'ignore-conditions': mapIgnoreConditions(update.ignore), 'security-updates-only': false, // TODO: update.'security-updates-only' 'security-advisories': [], // TODO: update.securityAdvisories source: { @@ -55,7 +55,7 @@ export class DependabotJobBuilder { } // Create a dependabot update job that updates a single pull request - static updatePullRequestJob( + static newUpdatePullRequestJob( variables: ISharedVariables, update: IDependabotUpdate, registries: Record, @@ -64,7 +64,7 @@ export class DependabotJobBuilder { ): IDependabotUpdateJob { const dependencyGroupName = pullRequestToUpdate['dependency-group-name']; const dependencies = (dependencyGroupName ? pullRequestToUpdate['dependencies'] : pullRequestToUpdate)?.map(d => d['dependency-name']); - const result = this.updateAllDependenciesJob(variables, update, registries, existingPullRequests); + const result = this.newUpdateAllJob(variables, update, registries, existingPullRequests); result.job['id'] = `update-${update.packageEcosystem}-${Date.now()}`; // TODO: Refine this result.job['updating-a-pull-request'] = true; result.job['dependency-group-to-refresh'] = dependencyGroupName; @@ -107,6 +107,9 @@ function mapRegistryCredentials(variables: ISharedVariables, registries: Record< // Map dependency groups function mapDependencyGroups(groups: string): any[] { + if (!groups) { + return []; + } const dependencyGroups = JSON.parse(groups); return Object.keys(dependencyGroups).map(name => { return { @@ -115,3 +118,11 @@ function mapDependencyGroups(groups: string): any[] { }; }); } + +// Map ignore conditions +function mapIgnoreConditions(ignore: string): any[] { + if (!ignore) { + return []; + } + return JSON.parse(ignore); +} diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index 845fb942..6ce6071d 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -4,12 +4,14 @@ import { IDependabotUpdateJob } from "./interfaces/IDependabotUpdateJob"; import { IDependabotUpdateOutputProcessor } from "./interfaces/IDependabotUpdateOutputProcessor"; import { AzureDevOpsWebApiClient } from "../azure-devops/AzureDevOpsWebApiClient"; import { GitPullRequestMergeStrategy, VersionControlChangeType } from "azure-devops-node-api/interfaces/GitInterfaces"; +import { IPullRequestProperties } from "../azure-devops/interfaces/IPullRequestProperties"; import * as path from 'path'; import * as crypto from 'crypto'; // Processes dependabot update outputs using the DevOps API export class DependabotOutputProcessor implements IDependabotUpdateOutputProcessor { private readonly api: AzureDevOpsWebApiClient; + private readonly existingPullRequests: IPullRequestProperties[]; private readonly taskVariables: ISharedVariables; // Custom properties used to store dependabot metadata in pull requests. @@ -17,15 +19,18 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess public static PR_PROPERTY_NAME_PACKAGE_MANAGER = "Dependabot.PackageManager"; public static PR_PROPERTY_NAME_DEPENDENCIES = "Dependabot.Dependencies"; - constructor(api: AzureDevOpsWebApiClient, taskVariables: ISharedVariables) { + constructor(api: AzureDevOpsWebApiClient, existingPullRequests: IPullRequestProperties[], taskVariables: ISharedVariables) { this.api = api; + this.existingPullRequests = existingPullRequests; this.taskVariables = taskVariables; } // Process the appropriate DevOps API actions for the supplied dependabot update output public async process(update: IDependabotUpdateJob, type: string, data: any): Promise { console.debug(`Processing output '${type}' with data:`, data); - let success: boolean = true; + const sourceRepoParts = update.job.source.repo.split('/'); // "{organisation}/{project}/_git/{repository}"" + const project = sourceRepoParts[1]; + const repository = sourceRepoParts[3]; switch (type) { // Documentation on the 'data' model for each output type can be found here: @@ -35,37 +40,24 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess // TODO: Store dependency list info in DevOps project properties? // This could be used to generate a dependency graph hub/page/report (future feature maybe?) // https://learn.microsoft.com/en-us/rest/api/azure/devops/core/projects/set-project-properties - break; + return true; case 'create_pull_request': if (this.taskVariables.skipPullRequests) { warning(`Skipping pull request creation as 'skipPullRequests' is set to 'true'`); - return; + return true; } // TODO: Skip if active pull request limit reached. - const sourceRepoParts = update.job.source.repo.split('/'); // "{organisation}/{project}/_git/{repository}"" - const dependencyGroupName = data['dependency-group']?.['name']; - let dependencies: any = data['dependencies']?.map((dep) => { - return { - 'dependency-name': dep['name'], - 'dependency-version': dep['version'], - 'directory': dep['directory'], - }; - }); - if (dependencyGroupName) { - dependencies = { - 'dependency-group-name': dependencyGroupName, - 'dependencies': dependencies - }; - } - await this.api.createPullRequest({ - project: sourceRepoParts[1], - repository: sourceRepoParts[3], + // Create a new pull request + const dependencies = getPullRequestDependenciesPropertyValueForOutputData(data); + const newPullRequest = await this.api.createPullRequest({ + project: project, + repository: repository, source: { commit: data['base-commit-sha'] || update.job.source.commit, - branch: sourceBranchNameForUpdate(update.job["package-manager"], update.config.targetBranch, dependencies) + branch: getSourceBranchNameForUpdate(update.job["package-manager"], update.config.targetBranch, dependencies) }, target: { branch: update.config.targetBranch @@ -116,12 +108,13 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess } ] }) - break; + + return newPullRequest !== undefined; case 'update_pull_request': if (this.taskVariables.skipPullRequests) { warning(`Skipping pull request update as 'skipPullRequests' is set to 'true'`); - return; + return true; } // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: update_pull_request() /* @@ -147,25 +140,33 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess Mode string `json:"mode" yaml:"mode,omitempty"` } */ - break; + return true; case 'close_pull_request': if (this.taskVariables.abandonUnwantedPullRequests) { warning(`Skipping pull request closure as 'abandonUnwantedPullRequests' is set to 'true'`); - return; + return true; } - // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: close_pull_request() - /* - type ClosePullRequest struct { - DependencyNames []string `json:"dependency-names" yaml:"dependency-names"` - Reason string `json:"reason" yaml:"reason"` + + // Find the pull request to close + const pullRequestToClose = this.getPullRequestForDependencyNames(update.job["package-manager"], data['dependency-names']); + if (!pullRequestToClose) { + warning(`Could not find pull request to close for package manager '${update.job["package-manager"]}' and dependencies '${data['dependency-names'].join(', ')}'`); + return true; } - */ - break; + + // Close the pull request + return await this.api.closePullRequest({ + project: project, + repository: repository, + pullRequestId: pullRequestToClose.id, + comment: this.taskVariables.commentPullRequests ? getPullRequestCloseReasonForOutputData(data) : undefined, + deleteSourceBranch: true + }); case 'mark_as_processed': // No action required - break; + return true; case 'record_ecosystem_versions': // No action required @@ -174,29 +175,33 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess case 'record_update_job_error': error(`Update job error: ${data['error-type']}`); console.log(data['error-details']); - success = false; - break; + return false; case 'record_update_job_unknown_error': error(`Update job unknown error: ${data['error-type']}`); console.log(data['error-details']); - success = false; - break; + return false; case 'increment_metric': // No action required - break; + return true; default: warning(`Unknown dependabot output type '${type}', ignoring...`); - break; + return true; } + } - return success; + private getPullRequestForDependencyNames(packageManager: string, dependencyNames: string[]): IPullRequestProperties | undefined { + return this.existingPullRequests.find(pr => { + return pr.properties.find(p => p.name === DependabotOutputProcessor.PR_PROPERTY_NAME_PACKAGE_MANAGER && p.value === packageManager) + && pr.properties.find(p => p.name === DependabotOutputProcessor.PR_PROPERTY_NAME_DEPENDENCIES && dependencyNamesAreEqual(getDependencyNames(JSON.parse(p.value)), dependencyNames)); + }); } + } -function sourceBranchNameForUpdate(packageEcosystem: string, targetBranch: string, dependencies: any): string { +function getSourceBranchNameForUpdate(packageEcosystem: string, targetBranch: string, dependencies: any): string { const target = targetBranch?.replace(/^\/+|\/+$/g, ''); // strip leading/trailing slashes if (dependencies['dependency-group-name']) { // Group dependency update @@ -212,3 +217,49 @@ function sourceBranchNameForUpdate(packageEcosystem: string, targetBranch: strin return `dependabot/${packageEcosystem}/${target}/${leadDependency['dependency-name']}-${leadDependency['dependency-version']}`; } } + +function getPullRequestCloseReasonForOutputData(data: any): string { + // The first dependency is the "lead" dependency in a multi-dependency update + const leadDependencyName = data['dependency-names'][0]; + let reason: string = null; + switch (data['reason']) { + case 'dependencies_changed': reason = `Looks like the dependencies have changed`; break; + case 'dependency_group_empty': reason = `Looks like the dependencies in this group are now empty`; break; + case 'dependency_removed': reason = `Looks like ${leadDependencyName} is no longer a dependency`; break; + case 'up_to_date': reason = `Looks like ${leadDependencyName} is up-to-date now`; break; + case 'update_no_longer_possible': reason = `Looks like ${leadDependencyName} can no longer be updated`; break; + // TODO: ??? => "Looks like these dependencies are updatable in another way, so this is no longer needed" + // TODO: ??? => "Superseded by ${new_pull_request_id}" + } + if (reason?.length > 0) { + reason += ', so this is no longer needed.'; + } + return reason; +} + +function getPullRequestDependenciesPropertyValueForOutputData(data: any): any { + const dependencyGroupName = data['dependency-group']?.['name']; + let dependencies: any = data['dependencies']?.map((dep) => { + return { + 'dependency-name': dep['name'], + 'dependency-version': dep['version'], + 'directory': dep['directory'], + }; + }); + if (dependencyGroupName) { + dependencies = { + 'dependency-group-name': dependencyGroupName, + 'dependencies': dependencies + }; + } + return dependencies; +} + +function getDependencyNames(dependencies: any): string[] { + return (dependencies['dependency-group-name'] ? dependencies['dependencies'] : dependencies)?.map((dep) => dep['dependency-name']?.toString()); +} + +function dependencyNamesAreEqual(a: string[], b: string[]): boolean { + if (a.length !== b.length) return false; + return a.every((name) => b.includes(name)); +} diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOutput.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOutput.ts index 972a25e7..ee40ca75 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOutput.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOutput.ts @@ -1,7 +1,7 @@ export interface IDependabotUpdateOutput { success: boolean, - error: any, + error: Error, output: { type: string, data: any From 21c5a71da1044dd91161e20442661aa080a795a5 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sat, 7 Sep 2024 22:50:32 +1200 Subject: [PATCH 19/57] Implement updating pull requests --- .../tasks/dependabot/dependabotV2/index.ts | 12 +- .../azure-devops/AzureDevOpsWebApiClient.ts | 90 ++++++++++++- .../azure-devops/interfaces/IPullRequest.ts | 14 +- .../DependabotOutputProcessor.ts | 124 ++++++++++-------- 4 files changed, 164 insertions(+), 76 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index 5f84140a..566a7595 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -2,7 +2,7 @@ import { which, setResult, TaskResult } from "azure-pipelines-task-lib/task" import { debug, warning, error } from "azure-pipelines-task-lib/task" import { DependabotCli } from './utils/dependabot-cli/DependabotCli'; import { AzureDevOpsWebApiClient } from "./utils/azure-devops/AzureDevOpsWebApiClient"; -import { DependabotOutputProcessor } from "./utils/dependabot-cli/DependabotOutputProcessor"; +import { DependabotOutputProcessor, parsePullRequestProperties } from "./utils/dependabot-cli/DependabotOutputProcessor"; import { DependabotJobBuilder } from "./utils/dependabot-cli/DependabotJobBuilder"; import { parseConfigFile } from './utils/parseConfigFile'; import getSharedVariables from './utils/getSharedVariables'; @@ -52,15 +52,7 @@ async function run() { // Parse the Dependabot metadata for the existing pull requests that are related to this update // Dependabot will use this to determine if we need to create new pull requests or update/close existing ones - const existingPullRequests = myActivePullRequests - .filter(pr => { - return pr.properties.find(p => p.name === DependabotOutputProcessor.PR_PROPERTY_NAME_PACKAGE_MANAGER && p.value === update.packageEcosystem); - }) - .map(pr => { - return JSON.parse( - pr.properties.find(p => p.name === DependabotOutputProcessor.PR_PROPERTY_NAME_DEPENDENCIES)?.value - ) - }); + const existingPullRequests = parsePullRequestProperties(myActivePullRequests, update.packageEcosystem); // Run an update job for "all dependencies"; this will create new pull requests for dependencies that need updating const allDependenciesJob = DependabotJobBuilder.newUpdateAllJob(taskVariables, update, dependabotConfig.registries, existingPullRequests); diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts index a6137959..78777596 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts @@ -1,8 +1,8 @@ import { debug, warning, error } from "azure-pipelines-task-lib/task" import { WebApi, getPersonalAccessTokenHandler } from "azure-devops-node-api"; -import { CommentThreadStatus, CommentType, ItemContentType, PullRequestStatus } from "azure-devops-node-api/interfaces/GitInterfaces"; +import { CommentThreadStatus, CommentType, ItemContentType, PullRequestAsyncStatus, PullRequestStatus } from "azure-devops-node-api/interfaces/GitInterfaces"; import { IPullRequestProperties } from "./interfaces/IPullRequestProperties"; -import { IPullRequest } from "./interfaces/IPullRequest"; +import { IPullRequest, IFileChange } from "./interfaces/IPullRequest"; // Wrapper for DevOps WebApi client with helper methods for easier management of dependabot pull requests export class AzureDevOpsWebApiClient { @@ -33,7 +33,7 @@ export class AzureDevOpsWebApiClient { ); return await Promise.all( - pullRequests.map(async pr => { + pullRequests?.map(async pr => { const properties = (await git.getPullRequestProperties(repository, pr.pullRequestId, project))?.value; return { id: pr.pullRequestId, @@ -81,8 +81,8 @@ export class AzureDevOpsWebApiClient { path: normalizeDevOpsPath(change.path) }, newContent: { - content: change.content, - contentType: ItemContentType.RawText + content: Buffer.from(change.content, change.encoding).toString('base64'), + contentType: ItemContentType.Base64Encoded } }; }) @@ -125,6 +125,10 @@ export class AzureDevOpsWebApiClient { ); } + // TODO: Upload the pull request description as a 'changes.md' file attachment? + // This might be a way to work around the 4000 character limit for PR descriptions, but needs more investigation. + // https://learn.microsoft.com/en-us/rest/api/azure/devops/git/pull-request-attachments/create?view=azure-devops-rest-7.1 + // Set the pull request auto-complete status if (pr.autoComplete) { console.info(` - Setting auto-complete...`); @@ -173,6 +177,82 @@ export class AzureDevOpsWebApiClient { } } + // Update a pull request + public async updatePullRequest(options: { + project: string, + repository: string, + pullRequestId: number, + changes: IFileChange[], + skipIfCommitsFromUsersOtherThan?: string, + skipIfNoConflicts?: boolean + }): Promise { + console.info(`Updating pull request #${options.pullRequestId}...`); + try { + const userId = await this.getUserId(); + const git = await this.connection.getGitApi(); + + // Get the pull request details + const pullRequest = await git.getPullRequest(options.repository, options.pullRequestId, options.project); + if (!pullRequest) { + throw new Error(`Pull request #${options.pullRequestId} not found`); + } + + // Skip if no merge conflicts + if (options.skipIfNoConflicts && pullRequest.mergeStatus !== PullRequestAsyncStatus.Conflicts) { + console.info(` - Skipping update as pull request has no merge conflicts.`); + return true; + } + + // Skip if the pull request has been modified by another user + const commits = await git.getPullRequestCommits(options.repository, options.pullRequestId, options.project); + if (options.skipIfCommitsFromUsersOtherThan && commits.some(c => c.author?.email !== options.skipIfCommitsFromUsersOtherThan)) { + console.info(` - Skipping update as pull request has been modified by another user.`); + return true; + } + + // Push changes to the source branch + console.info(` - Pushing ${options.changes.length} change(s) branch '${pullRequest.sourceRefName}'...`); + const push = await git.createPush( + { + refUpdates: [ + { + name: pullRequest.sourceRefName, + oldObjectId: pullRequest.lastMergeSourceCommit.commitId + } + ], + commits: [ + { + comment: (pullRequest.mergeStatus === PullRequestAsyncStatus.Conflicts) + ? "Resolve merge conflicts" + : "Update dependency files", + changes: options.changes.map(change => { + return { + changeType: change.changeType, + item: { + path: normalizeDevOpsPath(change.path) + }, + newContent: { + content: Buffer.from(change.content, change.encoding).toString('base64'), + contentType: ItemContentType.Base64Encoded + } + }; + }) + } + ] + }, + options.repository, + options.project + ); + + console.info(` - Pull request #${options.pullRequestId} was updated successfully.`); + return true; + } + catch (e) { + error(`Failed to update pull request: ${e}`); + return false; + } + } + // Close a pull request public async closePullRequest(options: { project: string, diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts index 4660d4db..0fbda9f4 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts @@ -29,14 +29,16 @@ export interface IPullRequest { reviewers?: string[], labels?: string[], workItems?: number[], - changes: { - changeType: VersionControlChangeType, - path: string, - content: string, - encoding: string - }[], + changes: IFileChange[], properties?: { name: string, value: string }[] }; + +export interface IFileChange { + changeType: VersionControlChangeType, + path: string, + content: string, + encoding: string +} diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index 6ce6071d..4b3023fd 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -57,7 +57,7 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess repository: repository, source: { commit: data['base-commit-sha'] || update.job.source.commit, - branch: getSourceBranchNameForUpdate(update.job["package-manager"], update.config.targetBranch, dependencies) + branch: getSourceBranchNameForUpdate(update.job["package-manager"], targetBranch, dependencies) }, target: { branch: update.config.targetBranch @@ -81,32 +81,8 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess reviewers: update.config.reviewers, labels: update.config.labels?.split(',').map((label) => label.trim()) || [], workItems: update.config.milestone ? [Number(update.config.milestone)] : [], - changes: data['updated-dependency-files'].filter((file) => file['type'] === 'file').map((file) => { - let changeType = VersionControlChangeType.None; - if (file['deleted'] === true) { - changeType = VersionControlChangeType.Delete; - } else if (file['operation'] === 'update') { - changeType = VersionControlChangeType.Edit; - } else { - changeType = VersionControlChangeType.Add; - } - return { - changeType: changeType, - path: path.join(file['directory'], file['name']), - content: file['content'], - encoding: file['content_encoding'] - } - }), - properties: [ - { - name: DependabotOutputProcessor.PR_PROPERTY_NAME_PACKAGE_MANAGER, - value: update.job["package-manager"] - }, - { - name: DependabotOutputProcessor.PR_PROPERTY_NAME_DEPENDENCIES, - value: JSON.stringify(dependencies) - } - ] + changes: getPullRequestChangedFilesForOutputData(data), + properties: buildPullRequestProperties(update.job["package-manager"], dependencies) }) return newPullRequest !== undefined; @@ -116,31 +92,25 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess warning(`Skipping pull request update as 'skipPullRequests' is set to 'true'`); return true; } - // TODO: Implement logic from /updater/lib/tinglesoftware/dependabot/api_clients/azure_apu_client.rb :: update_pull_request() - /* - type UpdatePullRequest struct { - BaseCommitSha string `json:"base-commit-sha" yaml:"base-commit-sha"` - DependencyNames []string `json:"dependency-names" yaml:"dependency-names"` - UpdatedDependencyFiles []DependencyFile `json:"updated-dependency-files" yaml:"updated-dependency-files"` - PRTitle string `json:"pr-title" yaml:"pr-title,omitempty"` - PRBody string `json:"pr-body" yaml:"pr-body,omitempty"` - CommitMessage string `json:"commit-message" yaml:"commit-message,omitempty"` - DependencyGroup map[string]any `json:"dependency-group" yaml:"dependency-group,omitempty"` - } - type DependencyFile struct { - Content string `json:"content" yaml:"content"` - ContentEncoding string `json:"content_encoding" yaml:"content_encoding"` - Deleted bool `json:"deleted" yaml:"deleted"` - Directory string `json:"directory" yaml:"directory"` - Name string `json:"name" yaml:"name"` - Operation string `json:"operation" yaml:"operation"` - SupportFile bool `json:"support_file" yaml:"support_file"` - SymlinkTarget string `json:"symlink_target,omitempty" yaml:"symlink_target,omitempty"` - Type string `json:"type" yaml:"type"` - Mode string `json:"mode" yaml:"mode,omitempty"` + + // Find the pull request to update + const pullRequestToUpdate = this.getPullRequestForDependencyNames(update.job["package-manager"], data['dependency-names']); + if (!pullRequestToUpdate) { + error(`Could not find pull request to update for package manager '${update.job["package-manager"]}' and dependencies '${data['dependency-names'].join(', ')}'`); + return false; } - */ - return true; + + // TODO: Skip if pull request has been manually modified (i.e. commits from other users) + + // Update the pull request + return await this.api.updatePullRequest({ + project: project, + repository: repository, + pullRequestId: pullRequestToUpdate.id, + changes: getPullRequestChangedFilesForOutputData(data), + skipIfCommitsFromUsersOtherThan: 'noreply@github.com', // TODO: this.taskVariables.extraEnvironmentVariables['DEPENDABOT_AUTHOR_EMAIL'] + skipIfNoConflicts: true, + }); case 'close_pull_request': if (this.taskVariables.abandonUnwantedPullRequests) { @@ -151,8 +121,8 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess // Find the pull request to close const pullRequestToClose = this.getPullRequestForDependencyNames(update.job["package-manager"], data['dependency-names']); if (!pullRequestToClose) { - warning(`Could not find pull request to close for package manager '${update.job["package-manager"]}' and dependencies '${data['dependency-names'].join(', ')}'`); - return true; + error(`Could not find pull request to close for package manager '${update.job["package-manager"]}' and dependencies '${data['dependency-names'].join(', ')}'`); + return false; } // Close the pull request @@ -195,12 +165,37 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess private getPullRequestForDependencyNames(packageManager: string, dependencyNames: string[]): IPullRequestProperties | undefined { return this.existingPullRequests.find(pr => { return pr.properties.find(p => p.name === DependabotOutputProcessor.PR_PROPERTY_NAME_PACKAGE_MANAGER && p.value === packageManager) - && pr.properties.find(p => p.name === DependabotOutputProcessor.PR_PROPERTY_NAME_DEPENDENCIES && dependencyNamesAreEqual(getDependencyNames(JSON.parse(p.value)), dependencyNames)); + && pr.properties.find(p => p.name === DependabotOutputProcessor.PR_PROPERTY_NAME_DEPENDENCIES && areEqual(getDependencyNames(JSON.parse(p.value)), dependencyNames)); }); } } +export function buildPullRequestProperties(packageManager: string, dependencies: any): any[] { + return [ + { + name: DependabotOutputProcessor.PR_PROPERTY_NAME_PACKAGE_MANAGER, + value: packageManager + }, + { + name: DependabotOutputProcessor.PR_PROPERTY_NAME_DEPENDENCIES, + value: JSON.stringify(dependencies) + } + ]; +} + +export function parsePullRequestProperties(pullRequests: IPullRequestProperties[], packageManager: string | null): any[] { + return pullRequests + .filter(pr => { + return pr.properties.find(p => p.name === DependabotOutputProcessor.PR_PROPERTY_NAME_PACKAGE_MANAGER && (packageManager === null || p.value === packageManager)); + }) + .map(pr => { + return JSON.parse( + pr.properties.find(p => p.name === DependabotOutputProcessor.PR_PROPERTY_NAME_DEPENDENCIES)?.value + ) + }); +} + function getSourceBranchNameForUpdate(packageEcosystem: string, targetBranch: string, dependencies: any): string { const target = targetBranch?.replace(/^\/+|\/+$/g, ''); // strip leading/trailing slashes if (dependencies['dependency-group-name']) { @@ -218,6 +213,25 @@ function getSourceBranchNameForUpdate(packageEcosystem: string, targetBranch: st } } +function getPullRequestChangedFilesForOutputData(data: any): any { + return data['updated-dependency-files'].filter((file) => file['type'] === 'file').map((file) => { + let changeType = VersionControlChangeType.None; + if (file['deleted'] === true) { + changeType = VersionControlChangeType.Delete; + } else if (file['operation'] === 'update') { + changeType = VersionControlChangeType.Edit; + } else { + changeType = VersionControlChangeType.Add; + } + return { + changeType: changeType, + path: path.join(file['directory'], file['name']), + content: file['content'], + encoding: file['content_encoding'] + } + }); +} + function getPullRequestCloseReasonForOutputData(data: any): string { // The first dependency is the "lead" dependency in a multi-dependency update const leadDependencyName = data['dependency-names'][0]; @@ -259,7 +273,7 @@ function getDependencyNames(dependencies: any): string[] { return (dependencies['dependency-group-name'] ? dependencies['dependencies'] : dependencies)?.map((dep) => dep['dependency-name']?.toString()); } -function dependencyNamesAreEqual(a: string[], b: string[]): boolean { +function areEqual(a: string[], b: string[]): boolean { if (a.length !== b.length) return false; return a.every((name) => b.includes(name)); } From ad570910d140716d693517d2849f66303c6b906a Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sat, 7 Sep 2024 22:51:01 +1200 Subject: [PATCH 20/57] Use default branch name if target branch not configured --- .../azure-devops/AzureDevOpsWebApiClient.ts | 17 +++++++++++++++++ .../dependabot-cli/DependabotOutputProcessor.ts | 3 ++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts index 78777596..f5a1ea0f 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts @@ -17,6 +17,23 @@ export class AzureDevOpsWebApiClient { ); } + // Get the default branch for a repository + public async getDefaultBranch(project: string, repository: string): Promise { + try { + const git = await this.connection.getGitApi(); + const repo = await git.getRepository(repository, project); + if (!repo) { + throw new Error(`Repository '${project}/${repository}' not found`); + } + + return repo.defaultBranch; + } + catch (e) { + error(`Failed to get default branch for '${project}/${repository}': ${e}`); + throw e; + } + } + // Get the properties for all active pull request created by the current user public async getMyActivePullRequestProperties(project: string, repository: string): Promise { console.info(`Fetching active pull request properties in '${project}/${repository}'...`); diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index 4b3023fd..c76a6961 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -52,6 +52,7 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess // Create a new pull request const dependencies = getPullRequestDependenciesPropertyValueForOutputData(data); + const targetBranch = update.config.targetBranch || await this.api.getDefaultBranch(project, repository); const newPullRequest = await this.api.createPullRequest({ project: project, repository: repository, @@ -60,7 +61,7 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess branch: getSourceBranchNameForUpdate(update.job["package-manager"], targetBranch, dependencies) }, target: { - branch: update.config.targetBranch + branch: targetBranch }, author: { email: 'noreply@github.com', // TODO: this.taskVariables.extraEnvironmentVariables['DEPENDABOT_AUTHOR_EMAIL'] From f9ba274a6cb6db1001381322e04359735100f8f9 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sun, 8 Sep 2024 01:25:57 +1200 Subject: [PATCH 21/57] Implement approving pull requests --- .../tasks/dependabot/dependabotV2/index.ts | 16 +++--- .../azure-devops/AzureDevOpsWebApiClient.ts | 49 ++++++++++++------- .../azure-devops/interfaces/IPullRequest.ts | 4 -- .../DependabotOutputProcessor.ts | 48 ++++++++++++------ 4 files changed, 73 insertions(+), 44 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index 566a7595..0ea1bd94 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -30,20 +30,20 @@ async function run() { throw new Error('Failed to parse dependabot.yaml configuration file from the target repository'); } - // Initialise the DevOps API client - const azdoApi = new AzureDevOpsWebApiClient( - taskVariables.organizationUrl.toString(), taskVariables.systemAccessToken - ); + // Initialise the DevOps API clients + // There are two clients; one for authoring pull requests and one for auto-approving pull requests (if configured) + const prAuthorClient = new AzureDevOpsWebApiClient(taskVariables.organizationUrl.toString(), taskVariables.systemAccessToken); + const prApproverClient = taskVariables.autoApprove ? new AzureDevOpsWebApiClient(taskVariables.organizationUrl.toString(), taskVariables.autoApproveUserToken) : null; - // Fetch the active pull requests created by our user - const myActivePullRequests = await azdoApi.getMyActivePullRequestProperties( + // Fetch the active pull requests created by the author user + const prAuthorActivePullRequests = await prAuthorClient.getMyActivePullRequestProperties( taskVariables.project, taskVariables.repository ); // Initialise the Dependabot updater dependabot = new DependabotCli( DependabotCli.CLI_IMAGE_LATEST, // TODO: Add config for this? - new DependabotOutputProcessor(azdoApi, myActivePullRequests, taskVariables), + new DependabotOutputProcessor(taskVariables, prAuthorClient, prApproverClient, prAuthorActivePullRequests), taskVariables.debug ); @@ -52,7 +52,7 @@ async function run() { // Parse the Dependabot metadata for the existing pull requests that are related to this update // Dependabot will use this to determine if we need to create new pull requests or update/close existing ones - const existingPullRequests = parsePullRequestProperties(myActivePullRequests, update.packageEcosystem); + const existingPullRequests = parsePullRequestProperties(prAuthorActivePullRequests, update.packageEcosystem); // Run an update job for "all dependencies"; this will create new pull requests for dependencies that need updating const allDependenciesJob = DependabotJobBuilder.newUpdateAllJob(taskVariables, update, dependabotConfig.registries, existingPullRequests); diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts index f5a1ea0f..65fcec16 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts @@ -149,11 +149,10 @@ export class AzureDevOpsWebApiClient { // Set the pull request auto-complete status if (pr.autoComplete) { console.info(` - Setting auto-complete...`); - const autoCompleteUserId = pr.autoComplete.userId || userId; await git.updatePullRequest( { autoCompleteSetBy: { - id: autoCompleteUserId + id: userId }, completionOptions: { autoCompleteIgnoreConfigIds: pr.autoComplete.ignorePolicyConfigIds, @@ -169,22 +168,6 @@ export class AzureDevOpsWebApiClient { ); } - // Set the pull request auto-approve status - if (pr.autoApprove) { - console.info(` - Approving pull request...`); - const approverUserId = pr.autoApprove.userId || userId; - await git.createPullRequestReviewer( - { - vote: 10, // 10 - approved 5 - approved with suggestions 0 - no vote -5 - waiting for author -10 - rejected - isReapprove: true - }, - pr.repository, - pullRequest.pullRequestId, - approverUserId, - pr.project - ); - } - console.info(` - Pull request ${pullRequest.pullRequestId} was created successfully.`); return pullRequest.pullRequestId; } @@ -270,6 +253,36 @@ export class AzureDevOpsWebApiClient { } } + // Approve a pull request + public async approvePullRequest(options: { + project: string, + repository: string, + pullRequestId: number + }): Promise { + console.info(`Approving pull request #${options.pullRequestId}...`); + try { + const userId = await this.getUserId(); + const git = await this.connection.getGitApi(); + + // Approve the pull request + console.info(` - Approving pull request...`); + await git.createPullRequestReviewer( + { + vote: 10, // 10 - approved 5 - approved with suggestions 0 - no vote -5 - waiting for author -10 - rejected + isReapprove: true + }, + options.repository, + options.pullRequestId, + userId, + options.project + ); + } + catch (e) { + error(`Failed to approve pull request: ${e}`); + return false; + } + } + // Close a pull request public async closePullRequest(options: { project: string, diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts index 0fbda9f4..f2fdca70 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts @@ -18,13 +18,9 @@ export interface IPullRequest { description: string, commitMessage: string, autoComplete?: { - userId: string | undefined, ignorePolicyConfigIds?: number[], mergeStrategy?: GitPullRequestMergeStrategy }, - autoApprove?: { - userId: string | undefined - }, assignees?: string[], reviewers?: string[], labels?: string[], diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index c76a6961..91c8cab1 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -10,7 +10,8 @@ import * as crypto from 'crypto'; // Processes dependabot update outputs using the DevOps API export class DependabotOutputProcessor implements IDependabotUpdateOutputProcessor { - private readonly api: AzureDevOpsWebApiClient; + private readonly prAuthorClient: AzureDevOpsWebApiClient; + private readonly prApproverClient: AzureDevOpsWebApiClient; private readonly existingPullRequests: IPullRequestProperties[]; private readonly taskVariables: ISharedVariables; @@ -19,8 +20,13 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess public static PR_PROPERTY_NAME_PACKAGE_MANAGER = "Dependabot.PackageManager"; public static PR_PROPERTY_NAME_DEPENDENCIES = "Dependabot.Dependencies"; - constructor(api: AzureDevOpsWebApiClient, existingPullRequests: IPullRequestProperties[], taskVariables: ISharedVariables) { - this.api = api; + public static PR_DEFAULT_AUTHOR_EMAIL = "noreply@github.com"; + public static PR_DEFAULT_AUTHOR_NAME = "dependabot[bot]"; + + constructor(taskVariables: ISharedVariables, prAuthorClient: AzureDevOpsWebApiClient, prApproverClient: AzureDevOpsWebApiClient, existingPullRequests: IPullRequestProperties[]) { + this.taskVariables = taskVariables; + this.prAuthorClient = prAuthorClient; + this.prApproverClient = prApproverClient; this.existingPullRequests = existingPullRequests; this.taskVariables = taskVariables; } @@ -52,8 +58,8 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess // Create a new pull request const dependencies = getPullRequestDependenciesPropertyValueForOutputData(data); - const targetBranch = update.config.targetBranch || await this.api.getDefaultBranch(project, repository); - const newPullRequest = await this.api.createPullRequest({ + const targetBranch = update.config.targetBranch || await this.prAuthorClient.getDefaultBranch(project, repository); + const newPullRequestId = await this.prAuthorClient.createPullRequest({ project: project, repository: repository, source: { @@ -71,13 +77,9 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess description: data['pr-body'], commitMessage: data['commit-message'], autoComplete: this.taskVariables.setAutoComplete ? { - userId: undefined, // TODO: add config for this? ignorePolicyConfigIds: this.taskVariables.autoCompleteIgnoreConfigIds, mergeStrategy: GitPullRequestMergeStrategy[this.taskVariables.mergeStrategy as keyof typeof GitPullRequestMergeStrategy] } : undefined, - autoApprove: this.taskVariables.autoApprove ? { - userId: this.taskVariables.autoApproveUserToken // TODO: convert token to user id - } : undefined, assignees: update.config.assignees, reviewers: update.config.reviewers, labels: update.config.labels?.split(',').map((label) => label.trim()) || [], @@ -86,7 +88,16 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess properties: buildPullRequestProperties(update.job["package-manager"], dependencies) }) - return newPullRequest !== undefined; + // Auto-approve the pull request, if required + if (this.taskVariables.autoApprove && this.prApproverClient && newPullRequestId) { + await this.prApproverClient.approvePullRequest({ + project: project, + repository: repository, + pullRequestId: newPullRequestId + }); + } + + return newPullRequestId !== undefined; case 'update_pull_request': if (this.taskVariables.skipPullRequests) { @@ -101,10 +112,8 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess return false; } - // TODO: Skip if pull request has been manually modified (i.e. commits from other users) - // Update the pull request - return await this.api.updatePullRequest({ + const pullRequestWasUpdated = await this.prAuthorClient.updatePullRequest({ project: project, repository: repository, pullRequestId: pullRequestToUpdate.id, @@ -113,6 +122,17 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess skipIfNoConflicts: true, }); + // Re-approve the pull request, if required + if (this.taskVariables.autoApprove && this.prApproverClient && pullRequestWasUpdated) { + await this.prApproverClient.approvePullRequest({ + project: project, + repository: repository, + pullRequestId: pullRequestToUpdate.id + }); + } + + return pullRequestWasUpdated; + case 'close_pull_request': if (this.taskVariables.abandonUnwantedPullRequests) { warning(`Skipping pull request closure as 'abandonUnwantedPullRequests' is set to 'true'`); @@ -127,7 +147,7 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess } // Close the pull request - return await this.api.closePullRequest({ + return await this.prAuthorClient.closePullRequest({ project: project, repository: repository, pullRequestId: pullRequestToClose.id, From f49bdae1e48011b8f4a221bca50c895fe5a3b82a Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sun, 8 Sep 2024 01:32:23 +1200 Subject: [PATCH 22/57] Add task inputs for pr commit author email and name --- .../tasks/dependabot/dependabotV2/task.json | 18 ++++++++++++++++++ .../DependabotOutputProcessor.ts | 6 +++--- .../dependabotV2/utils/getSharedVariables.ts | 9 +++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/task.json b/extension/tasks/dependabot/dependabotV2/task.json index 5096b307..bc45113d 100644 --- a/extension/tasks/dependabot/dependabotV2/task.json +++ b/extension/tasks/dependabot/dependabotV2/task.json @@ -75,6 +75,24 @@ "required": false, "helpMarkDown": "When set to `true` pull requests that are no longer needed are closed at the tail end of the execution. Defaults to `false`." }, + { + "name": "authorEmail", + "type": "string", + "groupName": "pull_requests", + "label": "Author email address", + "defaultValue": "", + "required": false, + "helpMarkDown": "The email address to use as the git commit author of the pull requests. Defaults to 'noreply@github.com'." + }, + { + "name": "authorName", + "type": "string", + "groupName": "pull_requests", + "label": "Author name", + "defaultValue": "", + "required": false, + "helpMarkDown": "The name to use as the git commit author of the pull requests. Defaults to 'dependabot[bot]'." + }, { "name": "setAutoComplete", "type": "boolean", diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index 91c8cab1..ffa700bc 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -70,8 +70,8 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess branch: targetBranch }, author: { - email: 'noreply@github.com', // TODO: this.taskVariables.extraEnvironmentVariables['DEPENDABOT_AUTHOR_EMAIL'] - name: 'dependabot[bot]', // TODO: this.taskVariables.extraEnvironmentVariables['DEPENDABOT_AUTHOR_NAME'] + email: this.taskVariables.authorEmail || DependabotOutputProcessor.PR_DEFAULT_AUTHOR_EMAIL, + name: this.taskVariables.authorName || DependabotOutputProcessor.PR_DEFAULT_AUTHOR_NAME }, title: data['pr-title'], description: data['pr-body'], @@ -118,7 +118,7 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess repository: repository, pullRequestId: pullRequestToUpdate.id, changes: getPullRequestChangedFilesForOutputData(data), - skipIfCommitsFromUsersOtherThan: 'noreply@github.com', // TODO: this.taskVariables.extraEnvironmentVariables['DEPENDABOT_AUTHOR_EMAIL'] + skipIfCommitsFromUsersOtherThan: this.taskVariables.authorEmail || DependabotOutputProcessor.PR_DEFAULT_AUTHOR_EMAIL, skipIfNoConflicts: true, }); diff --git a/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts b/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts index 34100a26..e4be4d00 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts @@ -37,6 +37,9 @@ export interface ISharedVariables { /** The access token for Azure DevOps Repos */ systemAccessToken: string; + authorEmail?: string; + authorName?: string; + /** Determines if the pull requests that dependabot creates should have auto complete set */ setAutoComplete: boolean; /** Merge strategies which can be used to complete a pull request */ @@ -114,6 +117,9 @@ export default function getSharedVariables(): ISharedVariables { let systemAccessUser: string = tl.getInput('azureDevOpsUser'); let systemAccessToken: string = getAzureDevOpsAccessToken(); + let authorEmail: string | undefined = tl.getInput('authorEmail'); + let authorName: string | undefined = tl.getInput('authorName'); + // Prepare variables for auto complete let setAutoComplete = tl.getBoolInput('setAutoComplete', false); let mergeStrategy = tl.getInput('mergeStrategy', true); @@ -165,6 +171,9 @@ export default function getSharedVariables(): ISharedVariables { systemAccessUser, systemAccessToken, + authorEmail, + authorName, + setAutoComplete, mergeStrategy, autoCompleteIgnoreConfigIds, From 85d28a7dfd395361181ff7d88aaf42bd802f658b Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sun, 8 Sep 2024 01:36:36 +1200 Subject: [PATCH 23/57] Implement open pull request limit config --- .../utils/dependabot-cli/DependabotOutputProcessor.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index ffa700bc..f7013b68 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -54,7 +54,11 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess return true; } - // TODO: Skip if active pull request limit reached. + // Skip if active pull request limit reached. + if (update.config.openPullRequestsLimit > 0 && this.existingPullRequests.length >= update.config.openPullRequestsLimit) { + warning(`Skipping pull request creation as the maximum number of active pull requests (${update.config.openPullRequestsLimit}) has been reached`); + return true; + } // Create a new pull request const dependencies = getPullRequestDependenciesPropertyValueForOutputData(data); From 021e5132be26f9e762def71f995c5930cb76b84b Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sun, 8 Sep 2024 01:49:47 +1200 Subject: [PATCH 24/57] Cleanup temporary files after task completion --- extension/tasks/dependabot/dependabotV2/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index 0ea1bd94..23982569 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -90,7 +90,7 @@ async function run() { exception(e); } finally { - // TODO: dependabotCli?.cleanup(); + dependabot?.cleanup(); } } From 5b2d29dfffbd6c0e1a4d9aec44da1ccac4769d58 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sun, 8 Sep 2024 01:50:14 +1200 Subject: [PATCH 25/57] Add configuration placeholders for dependabot component images --- extension/tasks/dependabot/dependabotV2/index.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index 23982569..b1e9f49f 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -47,16 +47,22 @@ async function run() { taskVariables.debug ); + const dependabotUpdaterOptions = { + collectorImage: undefined, // TODO: Add config for this? + proxyImage: undefined, // TODO: Add config for this? + updaterImage: undefined // TODO: Add config for this? + }; + // Loop through each 'update' block in dependabot.yaml and perform updates dependabotConfig.updates.forEach(async (update) => { - + // Parse the Dependabot metadata for the existing pull requests that are related to this update // Dependabot will use this to determine if we need to create new pull requests or update/close existing ones const existingPullRequests = parsePullRequestProperties(prAuthorActivePullRequests, update.packageEcosystem); // Run an update job for "all dependencies"; this will create new pull requests for dependencies that need updating const allDependenciesJob = DependabotJobBuilder.newUpdateAllJob(taskVariables, update, dependabotConfig.registries, existingPullRequests); - const allDependenciesUpdateOutputs = await dependabot.update(allDependenciesJob); + const allDependenciesUpdateOutputs = await dependabot.update(allDependenciesJob, dependabotUpdaterOptions); if (!allDependenciesUpdateOutputs || allDependenciesUpdateOutputs.filter(u => !u.success).length > 0) { allDependenciesUpdateOutputs.filter(u => !u.success).forEach(u => exception(u.error)); taskSucceeded = false; @@ -66,7 +72,7 @@ async function run() { if (!taskVariables.skipPullRequests) { for (const pr of existingPullRequests) { const updatePullRequestJob = DependabotJobBuilder.newUpdatePullRequestJob(taskVariables, update, dependabotConfig.registries, existingPullRequests, pr); - const updatePullRequestOutputs = await dependabot.update(updatePullRequestJob); + const updatePullRequestOutputs = await dependabot.update(updatePullRequestJob, dependabotUpdaterOptions); if (!updatePullRequestOutputs || updatePullRequestOutputs.filter(u => !u.success).length > 0) { updatePullRequestOutputs.filter(u => !u.success).forEach(u => exception(u.error)); taskSucceeded = false; From ebebd9fbfb7624af6b6b11e4ed1b2cca44ba3483 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Tue, 10 Sep 2024 00:34:45 +1200 Subject: [PATCH 26/57] Implement more config options --- .../tasks/dependabot/dependabotV2/index.ts | 35 +-- .../dependabotV2/utils/IDependabotConfig.ts | 146 ---------- .../azure-devops/AzureDevOpsWebApiClient.ts | 45 ++- .../azure-devops/interfaces/IFileChange.ts | 11 + .../azure-devops/interfaces/IPullRequest.ts | 13 +- .../interfaces/IPullRequestProperties.ts | 3 + .../utils/dependabot-cli/DependabotCli.ts | 48 ++-- .../dependabot-cli/DependabotJobBuilder.ts | 268 ++++++++++++------ .../DependabotOutputProcessor.ts | 29 +- ...teJob.ts => IDependabotUpdateJobConfig.ts} | 12 +- .../interfaces/IDependabotUpdateOperation.ts | 9 + .../IDependabotUpdateOperationResult.ts | 12 + .../interfaces/IDependabotUpdateOutput.ts | 9 - .../IDependabotUpdateOutputProcessor.ts | 15 +- .../interfaces/IDependabotConfig.ts | 105 +++++++ .../utils/{ => dependabot}/parseConfigFile.ts | 62 +--- 16 files changed, 462 insertions(+), 360 deletions(-) delete mode 100644 extension/tasks/dependabot/dependabotV2/utils/IDependabotConfig.ts create mode 100644 extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IFileChange.ts rename extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/{IDependabotUpdateJob.ts => IDependabotUpdateJobConfig.ts} (91%) create mode 100644 extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOperation.ts create mode 100644 extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOperationResult.ts delete mode 100644 extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOutput.ts create mode 100644 extension/tasks/dependabot/dependabotV2/utils/dependabot/interfaces/IDependabotConfig.ts rename extension/tasks/dependabot/dependabotV2/utils/{ => dependabot}/parseConfigFile.ts (79%) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index b1e9f49f..74dc74f6 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -4,8 +4,8 @@ import { DependabotCli } from './utils/dependabot-cli/DependabotCli'; import { AzureDevOpsWebApiClient } from "./utils/azure-devops/AzureDevOpsWebApiClient"; import { DependabotOutputProcessor, parsePullRequestProperties } from "./utils/dependabot-cli/DependabotOutputProcessor"; import { DependabotJobBuilder } from "./utils/dependabot-cli/DependabotJobBuilder"; -import { parseConfigFile } from './utils/parseConfigFile'; -import getSharedVariables from './utils/getSharedVariables'; +import parseDependabotConfigFile from './utils/dependabot/parseConfigFile'; +import parseTaskInputConfiguration from './utils/getSharedVariables'; async function run() { let taskSucceeded: boolean = true; @@ -19,32 +19,32 @@ async function run() { which('go', true); // Parse task input configuration - const taskVariables = getSharedVariables(); - if (!taskVariables) { + const taskInputs = parseTaskInputConfiguration(); + if (!taskInputs) { throw new Error('Failed to parse task input configuration'); } // Parse dependabot.yaml configuration file - const dependabotConfig = await parseConfigFile(taskVariables); + const dependabotConfig = await parseDependabotConfigFile(taskInputs); if (!dependabotConfig) { throw new Error('Failed to parse dependabot.yaml configuration file from the target repository'); } // Initialise the DevOps API clients // There are two clients; one for authoring pull requests and one for auto-approving pull requests (if configured) - const prAuthorClient = new AzureDevOpsWebApiClient(taskVariables.organizationUrl.toString(), taskVariables.systemAccessToken); - const prApproverClient = taskVariables.autoApprove ? new AzureDevOpsWebApiClient(taskVariables.organizationUrl.toString(), taskVariables.autoApproveUserToken) : null; + const prAuthorClient = new AzureDevOpsWebApiClient(taskInputs.organizationUrl.toString(), taskInputs.systemAccessToken); + const prApproverClient = taskInputs.autoApprove ? new AzureDevOpsWebApiClient(taskInputs.organizationUrl.toString(), taskInputs.autoApproveUserToken) : null; // Fetch the active pull requests created by the author user const prAuthorActivePullRequests = await prAuthorClient.getMyActivePullRequestProperties( - taskVariables.project, taskVariables.repository + taskInputs.project, taskInputs.repository ); // Initialise the Dependabot updater dependabot = new DependabotCli( DependabotCli.CLI_IMAGE_LATEST, // TODO: Add config for this? - new DependabotOutputProcessor(taskVariables, prAuthorClient, prApproverClient, prAuthorActivePullRequests), - taskVariables.debug + new DependabotOutputProcessor(taskInputs, prAuthorClient, prApproverClient, prAuthorActivePullRequests), + taskInputs.debug ); const dependabotUpdaterOptions = { @@ -54,14 +54,17 @@ async function run() { }; // Loop through each 'update' block in dependabot.yaml and perform updates - dependabotConfig.updates.forEach(async (update) => { + await Promise.all(dependabotConfig.updates.map(async (update) => { + // TODO: Read the last dependency list snapshot from project properties + const dependencyList = undefined; + // Parse the Dependabot metadata for the existing pull requests that are related to this update // Dependabot will use this to determine if we need to create new pull requests or update/close existing ones - const existingPullRequests = parsePullRequestProperties(prAuthorActivePullRequests, update.packageEcosystem); + const existingPullRequests = parsePullRequestProperties(prAuthorActivePullRequests, update["package-ecosystem"]); // Run an update job for "all dependencies"; this will create new pull requests for dependencies that need updating - const allDependenciesJob = DependabotJobBuilder.newUpdateAllJob(taskVariables, update, dependabotConfig.registries, existingPullRequests); + const allDependenciesJob = DependabotJobBuilder.newUpdateAllJob(taskInputs, update, dependabotConfig.registries, dependencyList, existingPullRequests); const allDependenciesUpdateOutputs = await dependabot.update(allDependenciesJob, dependabotUpdaterOptions); if (!allDependenciesUpdateOutputs || allDependenciesUpdateOutputs.filter(u => !u.success).length > 0) { allDependenciesUpdateOutputs.filter(u => !u.success).forEach(u => exception(u.error)); @@ -69,9 +72,9 @@ async function run() { } // Run an update job for each existing pull request; this will resolve merge conflicts and close pull requests that are no longer needed - if (!taskVariables.skipPullRequests) { + if (!taskInputs.skipPullRequests) { for (const pr of existingPullRequests) { - const updatePullRequestJob = DependabotJobBuilder.newUpdatePullRequestJob(taskVariables, update, dependabotConfig.registries, existingPullRequests, pr); + const updatePullRequestJob = DependabotJobBuilder.newUpdatePullRequestJob(taskInputs, update, dependabotConfig.registries, existingPullRequests, pr); const updatePullRequestOutputs = await dependabot.update(updatePullRequestJob, dependabotUpdaterOptions); if (!updatePullRequestOutputs || updatePullRequestOutputs.filter(u => !u.success).length > 0) { updatePullRequestOutputs.filter(u => !u.success).forEach(u => exception(u.error)); @@ -83,7 +86,7 @@ async function run() { return; } - }); + })); setResult( taskSucceeded ? TaskResult.Succeeded : TaskResult.Failed, diff --git a/extension/tasks/dependabot/dependabotV2/utils/IDependabotConfig.ts b/extension/tasks/dependabot/dependabotV2/utils/IDependabotConfig.ts deleted file mode 100644 index 714fe2c9..00000000 --- a/extension/tasks/dependabot/dependabotV2/utils/IDependabotConfig.ts +++ /dev/null @@ -1,146 +0,0 @@ -/** - * To see the supported structure, visit - * - * https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#configuration-options-for-dependabotyml - */ -export interface IDependabotConfig { - /** - * Mandatory. configuration file version. - **/ - version: number; - /** - * Mandatory. Configure how Dependabot updates the versions or project dependencies. - * Each entry configures the update settings for a particular package manager. - */ - updates: IDependabotUpdate[]; - /** - * Optional. Specify authentication details to access private package registries. - */ - registries?: Record; -} - -export interface IDependabotUpdate { - /** - * Package manager to use. - * */ - packageEcosystem: string; - /** - * Location of package manifests. - * https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#directory - * */ - directory: string; - /** - * Locations of package manifests. - * https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#directories - * */ - directories?: string[]; - /** - * Dependency group rules - * https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#groups - * */ - groups?: string; - /** - * Customize which updates are allowed. - */ - allow?: string; - /** - * Customize which updates are ignored. - */ - ignore?: string; - /** - * Custom labels/tags. - */ - labels?: string; - /** - * Reviewers. - */ - reviewers?: string[]; - /** - * Assignees. - */ - assignees?: string[]; - /** - * Commit Message. - */ - commitMessage?: string; - /** - * The milestone to associate pull requests with. - */ - milestone?: string; - /** - * Separator for the branches created. - */ - branchNameSeparator?: string; - /** - * Whether to reject external code - */ - insecureExternalCodeExecution?: string; - /** - * Limit number of open pull requests for version updates. - */ - openPullRequestsLimit?: number; - /** - * Registries configured for this update. - */ - registries: string[]; - /** - * Branch to create pull requests against. - */ - targetBranch?: string; - /** - * Update vendored or cached dependencies - */ - vendor?: boolean; - /** - * How to update manifest version requirements. - */ - versioningStrategy?: string; -} - -export interface IDependabotRegistry { - /** Identifies the type of registry*/ - 'type': string; - - /** - * The URL to use to access the dependencies. - * Dependabot adds or ignores trailing slashes as required. - * The protocol is optional. If not specified, `https://` is assumed. - */ - 'url'?: string | null | undefined; - 'index-url'?: string | null | undefined; // only for python_index - - /** - * The URL of the registry to use to access the dependencies. - * Dependabot adds or ignores trailing slashes as required. - * It should not have the scheme. - */ - 'registry'?: string | null | undefined; - /** The hostname for `terraform_registry` and `composer_repository` types */ - 'host'?: string | null | undefined; - - /** The username to access the registry */ - 'username'?: string | null | undefined; - /** A password for the username to access this registry */ - 'password'?: string | null | undefined; - /** An access key for this registry */ - 'key'?: string | null | undefined; - /** An access token for this registry */ - 'token'?: string | null | undefined; - - /** Organization for 'hex_organization' types */ - 'organization'?: string | null | undefined; - - /** Repository for 'hex_repository' types */ - 'repo'?: string | null | undefined; - /** Repository for 'hex_repository' types */ - 'auth-key'?: string | null | undefined; - /** Fingerprint of the public key for the Hex repository */ - 'public-key-fingerprint'?: string | null | undefined; - - /** - * For registries with type: python-index, - * if the boolean value is `true`, pip resolves dependencies by using the specified URL - * rather than the base URL of the Python Package Index (by default https://pypi.org/simple). - */ - 'replaces-base'?: boolean | null | undefined; -} diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts index 65fcec16..2db677ef 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts @@ -2,9 +2,12 @@ import { debug, warning, error } from "azure-pipelines-task-lib/task" import { WebApi, getPersonalAccessTokenHandler } from "azure-devops-node-api"; import { CommentThreadStatus, CommentType, ItemContentType, PullRequestAsyncStatus, PullRequestStatus } from "azure-devops-node-api/interfaces/GitInterfaces"; import { IPullRequestProperties } from "./interfaces/IPullRequestProperties"; -import { IPullRequest, IFileChange } from "./interfaces/IPullRequest"; +import { IPullRequest } from "./interfaces/IPullRequest"; +import { IFileChange } from "./interfaces/IFileChange"; -// Wrapper for DevOps WebApi client with helper methods for easier management of dependabot pull requests +/** + * Wrapper for DevOps WebApi client with helper methods for easier management of dependabot pull requests + */ export class AzureDevOpsWebApiClient { private readonly connection: WebApi; @@ -17,7 +20,12 @@ export class AzureDevOpsWebApiClient { ); } - // Get the default branch for a repository + /** + * Get the default branch for a repository + * @param project + * @param repository + * @returns + */ public async getDefaultBranch(project: string, repository: string): Promise { try { const git = await this.connection.getGitApi(); @@ -34,7 +42,12 @@ export class AzureDevOpsWebApiClient { } } - // Get the properties for all active pull request created by the current user + /** + * Get the properties for all active pull request created by the current user + * @param project + * @param repository + * @returns + */ public async getMyActivePullRequestProperties(project: string, repository: string): Promise { console.info(`Fetching active pull request properties in '${project}/${repository}'...`); try { @@ -70,7 +83,11 @@ export class AzureDevOpsWebApiClient { } } - // Create a new pull request + /** + * Create a new pull request + * @param pr + * @returns + */ public async createPullRequest(pr: IPullRequest): Promise { console.info(`Creating pull request '${pr.title}'...`); try { @@ -177,7 +194,11 @@ export class AzureDevOpsWebApiClient { } } - // Update a pull request + /** + * Update a pull request + * @param options + * @returns + */ public async updatePullRequest(options: { project: string, repository: string, @@ -253,7 +274,11 @@ export class AzureDevOpsWebApiClient { } } - // Approve a pull request + /** + * Approve a pull request + * @param options + * @returns + */ public async approvePullRequest(options: { project: string, repository: string, @@ -283,7 +308,11 @@ export class AzureDevOpsWebApiClient { } } - // Close a pull request + /** + * Close a pull request + * @param options + * @returns + */ public async closePullRequest(options: { project: string, repository: string, diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IFileChange.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IFileChange.ts new file mode 100644 index 00000000..bb2f06cf --- /dev/null +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IFileChange.ts @@ -0,0 +1,11 @@ +import { VersionControlChangeType } from "azure-devops-node-api/interfaces/TfvcInterfaces"; + +/** + * File change + */ +export interface IFileChange { + changeType: VersionControlChangeType, + path: string, + content: string, + encoding: string +} diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts index f2fdca70..99d5e6c4 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts @@ -1,5 +1,9 @@ -import { GitPullRequestMergeStrategy, VersionControlChangeType } from "azure-devops-node-api/interfaces/GitInterfaces"; +import { GitPullRequestMergeStrategy } from "azure-devops-node-api/interfaces/GitInterfaces"; +import { IFileChange } from "./IFileChange"; +/** + * Pull request creation + */ export interface IPullRequest { project: string, repository: string, @@ -31,10 +35,3 @@ export interface IPullRequest { value: string }[] }; - -export interface IFileChange { - changeType: VersionControlChangeType, - path: string, - content: string, - encoding: string -} diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequestProperties.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequestProperties.ts index bbc36433..d09c5ea3 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequestProperties.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequestProperties.ts @@ -1,4 +1,7 @@ +/** + * Pull request properties + */ export interface IPullRequestProperties { id: number, properties?: { diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts index 02140a4f..196625b5 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts @@ -2,14 +2,17 @@ import { debug, warning, error } from "azure-pipelines-task-lib/task" import { which, tool } from "azure-pipelines-task-lib/task" import { ToolRunner } from "azure-pipelines-task-lib/toolrunner" import { IDependabotUpdateOutputProcessor } from "./interfaces/IDependabotUpdateOutputProcessor"; -import { IDependabotUpdateOutput } from "./interfaces/IDependabotUpdateOutput"; -import { IDependabotUpdateJob } from "./interfaces/IDependabotUpdateJob"; +import { IDependabotUpdateOperationResult } from "./interfaces/IDependabotUpdateOperationResult"; +import { IDependabotUpdateOperation } from "./interfaces/IDependabotUpdateOperation"; import * as yaml from 'js-yaml'; import * as path from 'path'; import * as fs from 'fs'; import * as os from 'os'; +import { IDependabotUpdateJobConfig } from "./interfaces/IDependabotUpdateJobConfig"; -// Wrapper class for running updates using dependabot-cli +/** + * Wrapper class for running updates using dependabot-cli + */ export class DependabotCli { private readonly jobsPath: string; private readonly toolImage: string; @@ -26,21 +29,26 @@ export class DependabotCli { this.ensureJobsPathExists(); } - // Run dependabot update job + /** + * Run dependabot update job + * @param operation + * @param options + * @returns + */ public async update( - config: IDependabotUpdateJob, + operation: IDependabotUpdateOperation, options?: { collectorImage?: string, proxyImage?: string, updaterImage?: string } - ): Promise { + ): Promise { // Install dependabot if not already installed await this.ensureToolsAreInstalled(); // Create the job directory - const jobId = config.job.id; + const jobId = operation.job.id; const jobPath = path.join(this.jobsPath, jobId.toString()); const jobInputPath = path.join(jobPath, 'job.yaml'); const jobOutputPath = path.join(jobPath, 'scenario.yaml'); @@ -66,7 +74,7 @@ export class DependabotCli { } // Generate the job input file - writeJobFile(jobInputPath, config); + writeJobConfigFile(jobInputPath, operation); // Run dependabot update if (!fs.existsSync(jobOutputPath) || fs.statSync(jobOutputPath)?.size == 0) { @@ -83,17 +91,17 @@ export class DependabotCli { } // Process the job output - const processedOutputs = Array(); + const operationResults = Array(); if (fs.existsSync(jobOutputPath)) { - const outputs = readScenarioOutputs(jobOutputPath); - if (outputs?.length > 0) { + const jobOutputs = readJobScenarioOutputFile(jobOutputPath); + if (jobOutputs?.length > 0) { console.info("Processing Dependabot outputs..."); - for (const output of outputs) { + for (const output of jobOutputs) { // Documentation on the scenario model can be found here: // https://github.com/dependabot/cli/blob/main/internal/model/scenario.go const type = output['type']; const data = output['expect']?.['data']; - var processedOutput = { + var operationResult = { success: true, error: null, output: { @@ -102,20 +110,20 @@ export class DependabotCli { } }; try { - processedOutput.success = await this.outputProcessor.process(config, type, data); + operationResult.success = await this.outputProcessor.process(operation, type, data); } catch (e) { - processedOutput.success = false; - processedOutput.error = e; + operationResult.success = false; + operationResult.error = e; } finally { - processedOutputs.push(processedOutput); + operationResults.push(operationResult); } } } } - return processedOutputs.length > 0 ? processedOutputs : undefined; + return operationResults.length > 0 ? operationResults : undefined; } // Install dependabot if not already installed @@ -154,7 +162,7 @@ export class DependabotCli { // Documentation on the job model can be found here: // https://github.com/dependabot/cli/blob/main/internal/model/job.go -function writeJobFile(path: string, config: IDependabotUpdateJob): void { +function writeJobConfigFile(path: string, config: IDependabotUpdateJobConfig): void { fs.writeFileSync(path, yaml.dump({ job: config.job, credentials: config.credentials @@ -163,7 +171,7 @@ function writeJobFile(path: string, config: IDependabotUpdateJob): void { // Documentation on the scenario model can be found here: // https://github.com/dependabot/cli/blob/main/internal/model/scenario.go -function readScenarioOutputs(path: string): any[] { +function readJobScenarioOutputFile(path: string): any[] { const scenarioContent = fs.readFileSync(path, 'utf-8'); if (!scenarioContent || typeof scenarioContent !== 'string') { return []; // No outputs or failed scenario diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts index 8ee4743d..380715c9 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts @@ -1,88 +1,207 @@ +import { error, warning, debug } from "azure-pipelines-task-lib"; import { ISharedVariables } from "../getSharedVariables"; -import { IDependabotRegistry, IDependabotUpdate } from "../IDependabotConfig"; -import { IDependabotUpdateJob } from "./interfaces/IDependabotUpdateJob"; +import { IDependabotAllowCondition, IDependabotGroup, IDependabotRegistry, IDependabotUpdate } from "../dependabot/interfaces/IDependabotConfig"; +import { IDependabotUpdateOperation } from "./interfaces/IDependabotUpdateOperation"; +import * as crypto from 'crypto'; -// Wrapper class for building dependabot update job objects +/** + * Wrapper class for building dependabot update job objects + */ export class DependabotJobBuilder { - // Create a dependabot update job that updates all dependencies for a package ecyosystem + /** + * Create a dependabot update job that updates all dependencies for a package ecyosystem + * @param taskInputs + * @param update + * @param registries + * @param dependencyList + * @param existingPullRequests + * @returns + */ public static newUpdateAllJob( - variables: ISharedVariables, + taskInputs: ISharedVariables, update: IDependabotUpdate, registries: Record, + dependencyList: any[], existingPullRequests: any[] - ): IDependabotUpdateJob { - return { - config: update, - job: { - // TODO: Parse all options from `config` and `variables` - id: `update-${update.packageEcosystem}-all`, // TODO: Refine this - 'package-manager': update.packageEcosystem, - 'update-subdependencies': true, // TODO: add config option - 'updating-a-pull-request': false, - 'dependency-groups': mapDependencyGroups(update.groups), - 'allowed-updates': [ - { 'update-type': 'all' } // TODO: update.allow - ], - 'ignore-conditions': mapIgnoreConditions(update.ignore), - 'security-updates-only': false, // TODO: update.'security-updates-only' - 'security-advisories': [], // TODO: update.securityAdvisories - source: { - provider: 'azure', - 'api-endpoint': variables.apiEndpointUrl, - hostname: variables.hostname, - repo: `${variables.organization}/${variables.project}/_git/${variables.repository}`, - branch: update.targetBranch, - commit: undefined, // use latest commit of target branch - directory: update.directories?.length == 0 ? update.directory : undefined, - directories: update.directories?.length > 0 ? update.directories : undefined - }, - 'existing-pull-requests': existingPullRequests.filter(pr => !pr['dependency-group-name']), - 'existing-group-pull-requests': existingPullRequests.filter(pr => pr['dependency-group-name']), - 'commit-message-options': undefined, // TODO: update.commitMessageOptions - 'experiments': undefined, // TODO: update.experiments - 'max-updater-run-time': undefined, // TODO: update.maxUpdaterRunTime - 'reject-external-code': undefined, // TODO: update.insecureExternalCodeExecution - 'repo-private': undefined, // TODO: update.repoPrivate - 'repo-contents-path': undefined, // TODO: update.repoContentsPath - 'requirements-update-strategy': undefined, // TODO: update.requirementsUpdateStrategy - 'lockfile-only': undefined, // TODO: update.lockfileOnly - 'vendor-dependencies': undefined, // TODO: update.vendorDependencies - 'debug': variables.debug - }, - credentials: mapRegistryCredentials(variables, registries) - }; + ): IDependabotUpdateOperation { + const packageEcosystem = update["package-ecosystem"]; + const securityUpdatesOnly = update["open-pull-requests-limit"] == 0; + const updateDependencyNames = securityUpdatesOnly ? mapDependenciesForSecurityUpdate(dependencyList): undefined; + return buildUpdateJobConfig( + `update-${packageEcosystem}-${securityUpdatesOnly ? 'security-only' : 'all'}`, + taskInputs, + update, + registries, + false, + undefined, + updateDependencyNames, + existingPullRequests + ); } - // Create a dependabot update job that updates a single pull request - static newUpdatePullRequestJob( - variables: ISharedVariables, + /** + * Create a dependabot update job that updates a single pull request + * @param taskInputs + * @param update + * @param registries + * @param existingPullRequests + * @param pullRequestToUpdate + * @returns + */ + public static newUpdatePullRequestJob( + taskInputs: ISharedVariables, update: IDependabotUpdate, registries: Record, existingPullRequests: any[], pullRequestToUpdate: any - ): IDependabotUpdateJob { + ): IDependabotUpdateOperation { + const packageEcosystem = update["package-ecosystem"]; const dependencyGroupName = pullRequestToUpdate['dependency-group-name']; const dependencies = (dependencyGroupName ? pullRequestToUpdate['dependencies'] : pullRequestToUpdate)?.map(d => d['dependency-name']); - const result = this.newUpdateAllJob(variables, update, registries, existingPullRequests); - result.job['id'] = `update-${update.packageEcosystem}-${Date.now()}`; // TODO: Refine this - result.job['updating-a-pull-request'] = true; - result.job['dependency-group-to-refresh'] = dependencyGroupName; - result.job['dependencies'] = dependencies; - return result; + const dependencyNamesHash = crypto.createHash('md5').update(dependencies.join(',')).digest('hex').substring(0, 10) + return buildUpdateJobConfig( + `update-${packageEcosystem}-pr-${dependencyNamesHash}`, + taskInputs, + update, + registries, + true, + dependencyGroupName, + dependencies, + existingPullRequests + ); + } + +} + +function buildUpdateJobConfig( + id: string, + taskInputs: ISharedVariables, + update: IDependabotUpdate, + registries: Record, + updatingPullRequest: boolean, + updateDependencyGroupName: string | undefined, + updateDependencyNames: string[] | undefined, + existingPullRequests: any[]) { + const hasMultipleDirectories = update.directories?.length > 1; + return { + config: update, + job: { + 'id': id, + 'package-manager': update["package-ecosystem"], + 'update-subdependencies': true, // TODO: add config option? + 'updating-a-pull-request': updatingPullRequest, + 'dependency-group-to-refresh': updateDependencyGroupName, + 'dependency-groups': mapGroupsFromDependabotConfigToJobConfig(update.groups), + 'dependencies': updateDependencyNames, + 'allowed-updates': mapAllowedUpdatesFromDependabotConfigToJobConfig(update.allow), + 'ignore-conditions': mapIgnoreConditionsFromDependabotConfigToJobConfig(update.ignore), + 'security-updates-only': update["open-pull-requests-limit"] == 0, + 'security-advisories': [], // TODO: add config option + 'source': { + 'provider': 'azure', + 'api-endpoint': taskInputs.apiEndpointUrl, + 'hostname': taskInputs.hostname, + 'repo': `${taskInputs.organization}/${taskInputs.project}/_git/${taskInputs.repository}`, + 'branch': update["target-branch"], + 'commit': undefined, // use latest commit of target branch + 'directory': hasMultipleDirectories ? undefined : update.directory || '/', + 'directories': hasMultipleDirectories ? update.directories : undefined + }, + 'existing-pull-requests': existingPullRequests.filter(pr => !pr['dependency-group-name']), + 'existing-group-pull-requests': existingPullRequests.filter(pr => pr['dependency-group-name']), + 'commit-message-options': update["commit-message"] === undefined ? undefined : { + 'prefix': update["commit-message"]?.["prefix"], + 'prefix-development': update["commit-message"]?.["prefix-development"], + 'include-scope': update["commit-message"]?.["include"], + }, + 'experiments': undefined, // TODO: add config option + 'max-updater-run-time': undefined, // TODO: add config option? + 'reject-external-code': (update["insecure-external-code-execution"]?.toLocaleLowerCase() == "allow"), + 'repo-private': undefined, // TODO: add config option? + 'repo-contents-path': undefined, // TODO: add config option? + 'requirements-update-strategy': undefined, // TODO: add config option + 'lockfile-only': undefined, // TODO: add config option + 'vendor-dependencies': update.vendor, + 'debug': taskInputs.debug + }, + credentials: mapRegistryCredentialsFromDependabotConfigToJobConfig(taskInputs, registries) + }; +} + +function mapDependenciesForSecurityUpdate(dependencyList: any[]): string[] { + if (!dependencyList || dependencyList.length == 0) { + // No dependency list snapshot exists yet; Attempt to do a security update for all dependencies, but it will probably fail as it is not supported in dependabot-core yet + // TODO: Find a way to discover vulnerable dependencies for security-only updates without a dependency list snapshot. + // It would be nice if we could run dependabot-cli (e.g. `dependabot --discover-only`), but this is not supported currently. + warning( + "Security updates can only be performed if there is a previous dependency list snapshot available, but there is none as you have not completed a successful update job yet. " + + "Dependabot does not currently support discovering vulnerable dependencies during security-only updates and it is likely that this update operation will fail." + ); + return []; + } + return dependencyList.map(dependency => { + return dependency["dependency-name"]; + }); +} + +function mapGroupsFromDependabotConfigToJobConfig(dependencyGroups: Record): any[] { + if (!dependencyGroups) { + return undefined; + } + return Object.keys(dependencyGroups).map(name => { + const group = dependencyGroups[name]; + return { + 'name': name, + 'applies-to': group["applies-to"], + 'rules': { + 'patterns': group["patterns"], + 'exclude-patterns': group["exclude-patterns"], + 'dependency-type': group["dependency-type"], + 'update-types': group["update-types"] + } + }; + }); +} + +function mapAllowedUpdatesFromDependabotConfigToJobConfig(allowedUpdates: IDependabotAllowCondition[]): any[] { + if (!allowedUpdates) { + return [ + { 'dependency-type': 'all' } // if not explicitly configured, allow all updates + ]; } + return allowedUpdates.map(allow => { + return { + 'dependency-name': allow["dependency-name"], + 'dependency-type': allow["dependency-type"], + //'update-type': allow["update-type"] // TODO: This is missing from dependabot.ymal docs, but is used in the dependabot-core job model!? + }; + }); +} +function mapIgnoreConditionsFromDependabotConfigToJobConfig(ignoreConditions: IDependabotAllowCondition[]): any[] { + if (!ignoreConditions) { + return undefined; + } + return ignoreConditions.map(ignore => { + return { + 'dependency-name': ignore["dependency-name"], + //'source': ignore["source"], // TODO: This is missing from dependabot.ymal docs, but is used in the dependabot-core job model!? + 'update-types': ignore["update-types"], + //'updated-at': ignore["updated-at"], // TODO: This is missing from dependabot.ymal docs, but is used in the dependabot-core job model!? + 'version-requirement': (ignore["versions"])?.join(", "), // TODO: Test this, not sure how this should be parsed... + }; + }); } -// Map registry credentials -function mapRegistryCredentials(variables: ISharedVariables, registries: Record): any[] { +function mapRegistryCredentialsFromDependabotConfigToJobConfig(taskInputs: ISharedVariables, registries: Record): any[] { let registryCredentials = new Array(); - if (variables.systemAccessToken) { + if (taskInputs.systemAccessToken) { registryCredentials.push({ type: 'git_source', - host: variables.hostname, - username: variables.systemAccessUser?.trim()?.length > 0 ? variables.systemAccessUser : 'x-access-token', - password: variables.systemAccessToken + host: taskInputs.hostname, + username: taskInputs.systemAccessUser?.trim()?.length > 0 ? taskInputs.systemAccessUser : 'x-access-token', + password: taskInputs.systemAccessToken }); } if (registries) { @@ -93,36 +212,13 @@ function mapRegistryCredentials(variables: ISharedVariables, registries: Record< host: registry.host, url: registry.url, registry: registry.registry, - region: undefined, // TODO: registry.region, username: registry.username, password: registry.password, token: registry.token, - 'replaces-base': registry['replaces-base'] || false + 'replaces-base': registry["replaces-base"] }); } } return registryCredentials; } - -// Map dependency groups -function mapDependencyGroups(groups: string): any[] { - if (!groups) { - return []; - } - const dependencyGroups = JSON.parse(groups); - return Object.keys(dependencyGroups).map(name => { - return { - 'name': name, - 'rules': dependencyGroups[name] - }; - }); -} - -// Map ignore conditions -function mapIgnoreConditions(ignore: string): any[] { - if (!ignore) { - return []; - } - return JSON.parse(ignore); -} diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index f7013b68..b3e9a8f4 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -1,6 +1,6 @@ import { debug, warning, error } from "azure-pipelines-task-lib/task" import { ISharedVariables } from "../getSharedVariables"; -import { IDependabotUpdateJob } from "./interfaces/IDependabotUpdateJob"; +import { IDependabotUpdateOperation } from "./interfaces/IDependabotUpdateOperation"; import { IDependabotUpdateOutputProcessor } from "./interfaces/IDependabotUpdateOutputProcessor"; import { AzureDevOpsWebApiClient } from "../azure-devops/AzureDevOpsWebApiClient"; import { GitPullRequestMergeStrategy, VersionControlChangeType } from "azure-devops-node-api/interfaces/GitInterfaces"; @@ -8,7 +8,9 @@ import { IPullRequestProperties } from "../azure-devops/interfaces/IPullRequestP import * as path from 'path'; import * as crypto from 'crypto'; -// Processes dependabot update outputs using the DevOps API +/** + * Processes dependabot update outputs using the DevOps API + */ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcessor { private readonly prAuthorClient: AzureDevOpsWebApiClient; private readonly prApproverClient: AzureDevOpsWebApiClient; @@ -23,16 +25,22 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess public static PR_DEFAULT_AUTHOR_EMAIL = "noreply@github.com"; public static PR_DEFAULT_AUTHOR_NAME = "dependabot[bot]"; - constructor(taskVariables: ISharedVariables, prAuthorClient: AzureDevOpsWebApiClient, prApproverClient: AzureDevOpsWebApiClient, existingPullRequests: IPullRequestProperties[]) { - this.taskVariables = taskVariables; + constructor(taskInputs: ISharedVariables, prAuthorClient: AzureDevOpsWebApiClient, prApproverClient: AzureDevOpsWebApiClient, existingPullRequests: IPullRequestProperties[]) { + this.taskVariables = taskInputs; this.prAuthorClient = prAuthorClient; this.prApproverClient = prApproverClient; this.existingPullRequests = existingPullRequests; - this.taskVariables = taskVariables; + this.taskVariables = taskInputs; } - // Process the appropriate DevOps API actions for the supplied dependabot update output - public async process(update: IDependabotUpdateJob, type: string, data: any): Promise { + /** + * Process the appropriate DevOps API actions for the supplied dependabot update output + * @param update + * @param type + * @param data + * @returns + */ + public async process(update: IDependabotUpdateOperation, type: string, data: any): Promise { console.debug(`Processing output '${type}' with data:`, data); const sourceRepoParts = update.job.source.repo.split('/'); // "{organisation}/{project}/_git/{repository}"" const project = sourceRepoParts[1]; @@ -55,14 +63,15 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess } // Skip if active pull request limit reached. - if (update.config.openPullRequestsLimit > 0 && this.existingPullRequests.length >= update.config.openPullRequestsLimit) { - warning(`Skipping pull request creation as the maximum number of active pull requests (${update.config.openPullRequestsLimit}) has been reached`); + const openPullRequestLimit = update.config["open-pull-requests-limit"]; + if (openPullRequestLimit > 0 && this.existingPullRequests.length >= openPullRequestLimit) { + warning(`Skipping pull request creation as the maximum number of active pull requests (${openPullRequestLimit}) has been reached`); return true; } // Create a new pull request const dependencies = getPullRequestDependenciesPropertyValueForOutputData(data); - const targetBranch = update.config.targetBranch || await this.prAuthorClient.getDefaultBranch(project, repository); + const targetBranch = update.config["target-branch"] || await this.prAuthorClient.getDefaultBranch(project, repository); const newPullRequestId = await this.prAuthorClient.createPullRequest({ project: project, repository: repository, diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJob.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJobConfig.ts similarity index 91% rename from extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJob.ts rename to extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJobConfig.ts index 0e9c5443..64bb06a4 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJob.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJobConfig.ts @@ -1,10 +1,8 @@ -import { IDependabotUpdate } from "../../IDependabotConfig" -export interface IDependabotUpdateJob { - - // The raw dependabot.yaml update configuration options - // See: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file - config: IDependabotUpdate, +/** + * Represents the Dependabot CLI update job.yaml configuration file options. + */ +export interface IDependabotUpdateJobConfig { // The dependabot "updater" job configuration // See: https://github.com/dependabot/cli/blob/main/internal/model/job.go @@ -23,7 +21,7 @@ export interface IDependabotUpdateJob { 'exclude-patterns'?: string[], 'dependency-type'?: string 'update-types'?: string[] - }[] + } }[], 'dependencies'?: string[], 'allowed-updates'?: { diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOperation.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOperation.ts new file mode 100644 index 00000000..fe8e0eea --- /dev/null +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOperation.ts @@ -0,0 +1,9 @@ +import { IDependabotUpdate } from "../../dependabot/interfaces/IDependabotConfig" +import { IDependabotUpdateJobConfig } from "./IDependabotUpdateJobConfig" + +/** + * Represents a single Dependabot CLI update operation + */ +export interface IDependabotUpdateOperation extends IDependabotUpdateJobConfig { + config: IDependabotUpdate +} diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOperationResult.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOperationResult.ts new file mode 100644 index 00000000..3cc5afa0 --- /dev/null +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOperationResult.ts @@ -0,0 +1,12 @@ + +/** + * Represents the output of a Dependabot CLI update operation + */ +export interface IDependabotUpdateOperationResult { + success: boolean, + error: Error, + output: { + type: string, + data: any + } +} diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOutput.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOutput.ts deleted file mode 100644 index ee40ca75..00000000 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOutput.ts +++ /dev/null @@ -1,9 +0,0 @@ - -export interface IDependabotUpdateOutput { - success: boolean, - error: Error, - output: { - type: string, - data: any - } -} diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOutputProcessor.ts index df355221..7d37db9b 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateOutputProcessor.ts @@ -1,5 +1,16 @@ -import { IDependabotUpdateJob } from "./IDependabotUpdateJob"; +import { IDependabotUpdateOperation } from "./IDependabotUpdateOperation"; +/** + * Represents a processor for Dependabot update operation outputs + */ export interface IDependabotUpdateOutputProcessor { - process(update: IDependabotUpdateJob, type: string, data: any): Promise; + + /** + * Process the output of a Dependabot update operation + * @param update The update operation + * @param type The output type (e.g. "create-pull-request", "update-pull-request", etc.) + * @param data The output data object related to the type + */ + process(update: IDependabotUpdateOperation, type: string, data: any): Promise; + } diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot/interfaces/IDependabotConfig.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot/interfaces/IDependabotConfig.ts new file mode 100644 index 00000000..c4cb4ed5 --- /dev/null +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot/interfaces/IDependabotConfig.ts @@ -0,0 +1,105 @@ + +/** + * Represents the dependabot.yaml configuration file options. + * See: https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#configuration-options-for-dependabotyml + */ +export interface IDependabotConfig { + + /** + * Mandatory. configuration file version. + **/ + version: number, + + /** + * Mandatory. Configure how Dependabot updates the versions or project dependencies. + * Each entry configures the update settings for a particular package manager. + */ + updates: IDependabotUpdate[], + + /** + * Optional. + * Specify authentication details to access private package registries. + */ + registries?: Record + + /** + * Optional. Enables updates for ecosystems that are not yet generally available. + * https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#enable-beta-ecosystems + */ + 'enable-beta-ecosystems'?: boolean + +} + +export interface IDependabotUpdate { + 'package-ecosystem': string, + 'directory': string, + 'directories': string[], + 'allow'?: IDependabotAllowCondition[], + 'assignees'?: string[], + 'commit-message'?: IDependabotCommitMessage, + 'groups'?: Record, + 'ignore'?: IDependabotIgnoreCondition[], + 'insecure-external-code-execution'?: string, + 'labels'?: string, + 'milestone'?: string, + 'open-pull-requests-limit'?: number, + 'pull-request-branch-name'?: IDependabotPullRequestBranchName, + 'rebase-strategy'?: string, + 'registries'?: string[], + 'reviewers'?: string[], + 'schedule'?: IDependabotSchedule, + 'target-branch'?: string, + 'vendor'?: boolean, + 'versioning-strategy'?: string +} + +export interface IDependabotRegistry { + 'type': string, + 'url'?: string, + 'username'?: string, + 'password'?: string, + 'key'?: string, + 'token'?: string, + 'replaces-base'?: boolean, + 'host'?: string, // for terraform and composer only + 'registry'?: string, // for npm only + 'organization'?: string, // for hex-organisation only + 'repo'?: string, // for hex-repository only + 'public-key-fingerprint'?: string, // for hex-repository only +} + +export interface IDependabotGroup { + 'applies-to'?: string, + 'dependency-type'?: string, + 'patterns'?: string[], + 'exclude-patterns'?: string[], + 'update-types'?: string[] +} + +export interface IDependabotAllowCondition { + 'dependency-name'?: string, + 'dependency-type'?: string +} + +export interface IDependabotIgnoreCondition { + 'dependency-name'?: string, + 'versions'?: string[], + 'update-types'?: string[], +} + +export interface IDependabotSchedule { + 'interval'?: string, + 'day'?: string, + 'time'?: string, + 'timezone'?: string, +} + +export interface IDependabotCommitMessage { + 'prefix'?: string, + 'prefix-development'?: string, + 'include'?: string +} + +export interface IDependabotPullRequestBranchName { + 'separator'?: string +} diff --git a/extension/tasks/dependabot/dependabotV2/utils/parseConfigFile.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot/parseConfigFile.ts similarity index 79% rename from extension/tasks/dependabot/dependabotV2/utils/parseConfigFile.ts rename to extension/tasks/dependabot/dependabotV2/utils/dependabot/parseConfigFile.ts index 356aa7f5..8380e7f4 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/parseConfigFile.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot/parseConfigFile.ts @@ -5,26 +5,24 @@ import * as fs from 'fs'; import { load } from 'js-yaml'; import * as path from 'path'; import { URL } from 'url'; -import { IDependabotConfig, IDependabotRegistry, IDependabotUpdate } from './IDependabotConfig'; -import { convertPlaceholder } from './convertPlaceholder'; -import { ISharedVariables } from './getSharedVariables'; +import { IDependabotConfig, IDependabotRegistry, IDependabotUpdate } from './interfaces/IDependabotConfig'; +import { convertPlaceholder } from '../convertPlaceholder'; +import { ISharedVariables } from '../getSharedVariables'; /** - * Parse the dependabot config YAML file to specify update configuration - * + * Parse the dependabot config YAML file to specify update configuration. * The file should be located at '/.azuredevops/dependabot.yml' or '/.github/dependabot.yml' * * To view YAML file format, visit * https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#allow * - * @param variables the shared variables of the task + * @param taskInputs the input variables of the task * @returns {IDependabotConfig} config - the dependabot configuration */ -async function parseConfigFile(variables: ISharedVariables): Promise { +export default async function parseConfigFile(taskInputs: ISharedVariables): Promise { const possibleFilePaths = [ '/.azuredevops/dependabot.yml', '/.azuredevops/dependabot.yaml', - '/.github/dependabot.yaml', '/.github/dependabot.yml', ]; @@ -37,18 +35,18 @@ async function parseConfigFile(variables: ISharedVariables): Promise { - var dependabotUpdate: IDependabotUpdate = { - packageEcosystem: update['package-ecosystem'], - directory: update['directory'], - directories: update['directories'] || [], - - openPullRequestsLimit: update['open-pull-requests-limit'], - registries: update['registries'] || [], - - targetBranch: update['target-branch'], - vendor: update['vendor'] ? JSON.parse(update['vendor']) : null, - versioningStrategy: update['versioning-strategy'], - milestone: update['milestone'], - branchNameSeparator: update['pull-request-branch-name'] - ? update['pull-request-branch-name']['separator'] - : undefined, - insecureExternalCodeExecution: update['insecure-external-code-execution'], - - // We are well aware that ignore is not parsed here. It is intentional. - // The ruby script in the docker container does it automatically. - // If you are having issues, search for related issues such as https://github.com/tinglesoftware/dependabot-azure-devops/pull/582 - // before creating a new issue. - // You can also test against various reproductions such as https://dev.azure.com/tingle/dependabot/_git/repro-582 - - // Convert to JSON or as required by the script - allow: update['allow'] ? JSON.stringify(update['allow']) : undefined, - ignore: update['ignore'] ? JSON.stringify(update['ignore']) : undefined, - labels: update['labels'] ? JSON.stringify(update['labels']) : undefined, - reviewers: update['reviewers'] ? update['reviewers'] : undefined, - assignees: update['assignees'] ? update['assignees'] : undefined, - commitMessage: update['commit-message'] ? JSON.stringify(update['commit-message']) : undefined, - groups: update['groups'] ? JSON.stringify(update['groups']) : undefined, - }; - - if (!dependabotUpdate.packageEcosystem) { + var dependabotUpdate: IDependabotUpdate = update; + + if (!dependabotUpdate['package-ecosystem']) { throw new Error("The value 'package-ecosystem' in dependency update config is missing"); } // zero is a valid value - if (!dependabotUpdate.openPullRequestsLimit && dependabotUpdate.openPullRequestsLimit !== 0) { - dependabotUpdate.openPullRequestsLimit = 5; + if (!dependabotUpdate['open-pull-requests-limit'] && dependabotUpdate['open-pull-requests-limit'] !== 0) { + dependabotUpdate['open-pull-requests-limit'] = 5; } if (!dependabotUpdate.directory && dependabotUpdate.directories.length === 0) { @@ -336,4 +303,3 @@ const KnownRegistryTypes = [ 'terraform-registry', ]; -export { parseConfigFile, parseRegistries, parseUpdates, validateConfiguration }; From db2c6e8792b9fc9a5d16366426c10058576de1ca Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Tue, 10 Sep 2024 01:25:55 +1200 Subject: [PATCH 27/57] Implement dependency list snapshots, which are stored in the DevOps project properties --- .../tasks/dependabot/dependabotV2/index.ts | 14 ++-- .../azure-devops/AzureDevOpsWebApiClient.ts | 64 ++++++++++++++++++- .../dependabot-cli/DependabotJobBuilder.ts | 4 +- .../DependabotOutputProcessor.ts | 30 +++++++-- 4 files changed, 100 insertions(+), 12 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index 74dc74f6..4625f2a5 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -2,7 +2,7 @@ import { which, setResult, TaskResult } from "azure-pipelines-task-lib/task" import { debug, warning, error } from "azure-pipelines-task-lib/task" import { DependabotCli } from './utils/dependabot-cli/DependabotCli'; import { AzureDevOpsWebApiClient } from "./utils/azure-devops/AzureDevOpsWebApiClient"; -import { DependabotOutputProcessor, parsePullRequestProperties } from "./utils/dependabot-cli/DependabotOutputProcessor"; +import { DependabotOutputProcessor, parseDependencyListProperty, parsePullRequestProperties } from "./utils/dependabot-cli/DependabotOutputProcessor"; import { DependabotJobBuilder } from "./utils/dependabot-cli/DependabotJobBuilder"; import parseDependabotConfigFile from './utils/dependabot/parseConfigFile'; import parseTaskInputConfiguration from './utils/getSharedVariables'; @@ -56,15 +56,21 @@ async function run() { // Loop through each 'update' block in dependabot.yaml and perform updates await Promise.all(dependabotConfig.updates.map(async (update) => { - // TODO: Read the last dependency list snapshot from project properties - const dependencyList = undefined; + // Parse the last dependency list snapshot (if any) from the project properties. + // This is required when doing a security-only update as dependabot requires the list of vulnerable dependencies to be updated update. + // Automatic discovery of vulnerable dependencies during a security-only update is not currently supported by dependabot-updater. + const dependencyList = parseDependencyListProperty( + await prAuthorClient.getProjectProperty(taskInputs.project, DependabotOutputProcessor.PROJECT_PROPERTY_NAME_DEPENDENCY_LIST), + taskInputs.repository, + update["package-ecosystem"] + ); // Parse the Dependabot metadata for the existing pull requests that are related to this update // Dependabot will use this to determine if we need to create new pull requests or update/close existing ones const existingPullRequests = parsePullRequestProperties(prAuthorActivePullRequests, update["package-ecosystem"]); // Run an update job for "all dependencies"; this will create new pull requests for dependencies that need updating - const allDependenciesJob = DependabotJobBuilder.newUpdateAllJob(taskInputs, update, dependabotConfig.registries, dependencyList, existingPullRequests); + const allDependenciesJob = DependabotJobBuilder.newUpdateAllJob(taskInputs, update, dependabotConfig.registries, dependencyList['dependencies'], existingPullRequests); const allDependenciesUpdateOutputs = await dependabot.update(allDependenciesJob, dependabotUpdaterOptions); if (!allDependenciesUpdateOutputs || allDependenciesUpdateOutputs.filter(u => !u.success).length > 0) { allDependenciesUpdateOutputs.filter(u => !u.success).forEach(u => exception(u.error)); diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts index 2db677ef..1b9fa342 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts @@ -20,6 +20,10 @@ export class AzureDevOpsWebApiClient { ); } + private async getUserId(): Promise { + return (this.userId ||= (await this.connection.connect()).authenticatedUser?.id || ""); + } + /** * Get the default branch for a repository * @param project @@ -385,9 +389,65 @@ export class AzureDevOpsWebApiClient { return false; } } + + /** + * Get a project property + * @param project + * @param name + * @param valueBuilder + * @returns + */ + public async getProjectProperty(project: string, name: string): Promise { + try { + const core = await this.connection.getCoreApi(); + const projects = await core.getProjects(); + const projectGuid = projects?.find(p => p.name === project)?.id; + const properties = await core.getProjectProperties(projectGuid); + return properties?.find(p => p.name === name)?.value; + } catch (e) { + error(`Failed to get project property '${name}': ${e}`); + console.log(e); + return null; + } + } - private async getUserId(): Promise { - return (this.userId ||= (await this.connection.connect()).authenticatedUser?.id || ""); + /** + * Update a project property + * @param project + * @param name + * @param valueBuilder + * @returns + */ + public async updateProjectProperty(project: string, name: string, valueBuilder: (existingValue: string) => string): Promise { + try { + + // Get the existing project property value + const core = await this.connection.getCoreApi(); + const projects = await core.getProjects(); + const projectGuid = projects?.find(p => p.name === project)?.id; + const properties = await core.getProjectProperties(projectGuid); + const propertyValue = properties?.find(p => p.name === name)?.value; + + // Update the project property + await core.setProjectProperties( + undefined, + projectGuid, + [ + { + op: "add", + path: "/" + name, + value: valueBuilder(propertyValue || "") + } + ] + ); + + return true; + + } catch (e) { + error(`Failed to update project property '${name}': ${e}`); + console.log(e); + return false; + } } } diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts index 380715c9..43f4472b 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts @@ -131,7 +131,7 @@ function buildUpdateJobConfig( function mapDependenciesForSecurityUpdate(dependencyList: any[]): string[] { if (!dependencyList || dependencyList.length == 0) { - // No dependency list snapshot exists yet; Attempt to do a security update for all dependencies, but it will probably fail as it is not supported in dependabot-core yet + // No dependency list snapshot exists yet; Attempt to do a security update for all dependencies, but it will probably fail as it is not supported in dependabot-updater yet // TODO: Find a way to discover vulnerable dependencies for security-only updates without a dependency list snapshot. // It would be nice if we could run dependabot-cli (e.g. `dependabot --discover-only`), but this is not supported currently. warning( @@ -141,7 +141,7 @@ function mapDependenciesForSecurityUpdate(dependencyList: any[]): string[] { return []; } return dependencyList.map(dependency => { - return dependency["dependency-name"]; + return dependency["name"]; }); } diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index b3e9a8f4..284228b2 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -17,6 +17,10 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess private readonly existingPullRequests: IPullRequestProperties[]; private readonly taskVariables: ISharedVariables; + // Custom properties used to store dependabot metadata in projects. + // https://learn.microsoft.com/en-us/rest/api/azure/devops/core/projects/set-project-properties + public static PROJECT_PROPERTY_NAME_DEPENDENCY_LIST = "Dependabot.DependencyList"; + // Custom properties used to store dependabot metadata in pull requests. // https://learn.microsoft.com/en-us/rest/api/azure/devops/git/pull-request-properties public static PR_PROPERTY_NAME_PACKAGE_MANAGER = "Dependabot.PackageManager"; @@ -51,10 +55,23 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess // See: https://github.com/dependabot/cli/blob/main/internal/model/update.go case 'update_dependency_list': - // TODO: Store dependency list info in DevOps project properties? - // This could be used to generate a dependency graph hub/page/report (future feature maybe?) - // https://learn.microsoft.com/en-us/rest/api/azure/devops/core/projects/set-project-properties - return true; + + // Store the dependency list snapshot in project properties, for future reference + return this.prAuthorClient.updateProjectProperty( + project, + DependabotOutputProcessor.PROJECT_PROPERTY_NAME_DEPENDENCY_LIST, + function(existingValue: string) { + const repoDependencyLists = JSON.parse(existingValue || '{}'); + repoDependencyLists[repository] = repoDependencyLists[repository] || {}; + repoDependencyLists[repository][update.job["package-manager"]] = { + 'dependencies': data['dependencies'], + 'dependency-files': data['dependency_files'], + 'last-updated': new Date().toISOString() + }; + + return JSON.stringify(repoDependencyLists); + } + ); case 'create_pull_request': if (this.taskVariables.skipPullRequests) { @@ -218,6 +235,11 @@ export function buildPullRequestProperties(packageManager: string, dependencies: ]; } +export function parseDependencyListProperty(dependencyList: string, repository: string, packageManager: string): any { + const repoDependencyLists = JSON.parse(dependencyList || '{}'); + return repoDependencyLists[repository]?.[packageManager]; +} + export function parsePullRequestProperties(pullRequests: IPullRequestProperties[], packageManager: string | null): any[] { return pullRequests .filter(pr => { From 2ba73d498e048bd6aa2d1530577548a97715bf70 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Tue, 10 Sep 2024 19:49:28 +1200 Subject: [PATCH 28/57] Add task input option for storing dependency list --- .../tasks/dependabot/dependabotV2/index.ts | 10 +-- .../tasks/dependabot/dependabotV2/task.json | 9 +++ .../azure-devops/AzureDevOpsWebApiClient.ts | 20 ++--- .../dependabot-cli/DependabotJobBuilder.ts | 33 ++++---- .../DependabotOutputProcessor.ts | 77 ++++++++++--------- .../interfaces/IDependabotUpdateJobConfig.ts | 2 - .../dependabotV2/utils/getDockerImageTag.ts | 36 --------- .../dependabotV2/utils/getSharedVariables.ts | 39 ++-------- 8 files changed, 89 insertions(+), 137 deletions(-) delete mode 100644 extension/tasks/dependabot/dependabotV2/utils/getDockerImageTag.ts diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index 4625f2a5..d668f4fe 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -2,7 +2,7 @@ import { which, setResult, TaskResult } from "azure-pipelines-task-lib/task" import { debug, warning, error } from "azure-pipelines-task-lib/task" import { DependabotCli } from './utils/dependabot-cli/DependabotCli'; import { AzureDevOpsWebApiClient } from "./utils/azure-devops/AzureDevOpsWebApiClient"; -import { DependabotOutputProcessor, parseDependencyListProperty, parsePullRequestProperties } from "./utils/dependabot-cli/DependabotOutputProcessor"; +import { DependabotOutputProcessor, parseProjectDependencyListProperty, parsePullRequestProperties } from "./utils/dependabot-cli/DependabotOutputProcessor"; import { DependabotJobBuilder } from "./utils/dependabot-cli/DependabotJobBuilder"; import parseDependabotConfigFile from './utils/dependabot/parseConfigFile'; import parseTaskInputConfiguration from './utils/getSharedVariables'; @@ -33,7 +33,7 @@ async function run() { // Initialise the DevOps API clients // There are two clients; one for authoring pull requests and one for auto-approving pull requests (if configured) const prAuthorClient = new AzureDevOpsWebApiClient(taskInputs.organizationUrl.toString(), taskInputs.systemAccessToken); - const prApproverClient = taskInputs.autoApprove ? new AzureDevOpsWebApiClient(taskInputs.organizationUrl.toString(), taskInputs.autoApproveUserToken) : null; + const prApproverClient = taskInputs.autoApprove ? new AzureDevOpsWebApiClient(taskInputs.organizationUrl.toString(), taskInputs.autoApproveUserToken || taskInputs.systemAccessToken) : null; // Fetch the active pull requests created by the author user const prAuthorActivePullRequests = await prAuthorClient.getMyActivePullRequestProperties( @@ -57,10 +57,10 @@ async function run() { await Promise.all(dependabotConfig.updates.map(async (update) => { // Parse the last dependency list snapshot (if any) from the project properties. - // This is required when doing a security-only update as dependabot requires the list of vulnerable dependencies to be updated update. + // This is required when doing a security-only update as dependabot requires the list of vulnerable dependencies to be updated. // Automatic discovery of vulnerable dependencies during a security-only update is not currently supported by dependabot-updater. - const dependencyList = parseDependencyListProperty( - await prAuthorClient.getProjectProperty(taskInputs.project, DependabotOutputProcessor.PROJECT_PROPERTY_NAME_DEPENDENCY_LIST), + const dependencyList = parseProjectDependencyListProperty( + await prAuthorClient.getProjectProperties(taskInputs.project), taskInputs.repository, update["package-ecosystem"] ); diff --git a/extension/tasks/dependabot/dependabotV2/task.json b/extension/tasks/dependabot/dependabotV2/task.json index bc45113d..1fba3900 100644 --- a/extension/tasks/dependabot/dependabotV2/task.json +++ b/extension/tasks/dependabot/dependabotV2/task.json @@ -48,6 +48,15 @@ } ], "inputs": [ + { + "name": "storeDependencyList", + "type": "boolean", + "groupName": "advanced", + "label": "Monitor the discovered dependencies", + "defaultValue": false, + "required": false, + "helpMarkDown": "When set to `true`, the discovered dependencies will be stored against the project. Defaults to `false`." + }, { "name": "skipPullRequests", "type": "boolean", diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts index 1b9fa342..8a35af39 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts @@ -391,23 +391,23 @@ export class AzureDevOpsWebApiClient { } /** - * Get a project property + * Get project properties * @param project - * @param name * @param valueBuilder * @returns */ - public async getProjectProperty(project: string, name: string): Promise { + public async getProjectProperties(project: string): Promise | undefined> { try { const core = await this.connection.getCoreApi(); const projects = await core.getProjects(); const projectGuid = projects?.find(p => p.name === project)?.id; const properties = await core.getProjectProperties(projectGuid); - return properties?.find(p => p.name === name)?.value; + return properties + .map(p => ({ [p.name]: p.value })) + .reduce((a, b) => ({ ...a, ...b }), {}); + } catch (e) { - error(`Failed to get project property '${name}': ${e}`); - console.log(e); - return null; + error(`Failed to get project properties: ${e}`); } } @@ -418,7 +418,7 @@ export class AzureDevOpsWebApiClient { * @param valueBuilder * @returns */ - public async updateProjectProperty(project: string, name: string, valueBuilder: (existingValue: string) => string): Promise { + public async updateProjectProperty(project: string, name: string, valueBuilder: (existingValue: string) => string): Promise { try { // Get the existing project property value @@ -441,12 +441,8 @@ export class AzureDevOpsWebApiClient { ] ); - return true; - } catch (e) { error(`Failed to update project property '${name}': ${e}`); - console.log(e); - return false; } } } diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts index 43f4472b..43b01c7c 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts @@ -89,7 +89,7 @@ function buildUpdateJobConfig( job: { 'id': id, 'package-manager': update["package-ecosystem"], - 'update-subdependencies': true, // TODO: add config option? + 'update-subdependencies': true, // TODO: add config for this? 'updating-a-pull-request': updatingPullRequest, 'dependency-group-to-refresh': updateDependencyGroupName, 'dependency-groups': mapGroupsFromDependabotConfigToJobConfig(update.groups), @@ -97,7 +97,7 @@ function buildUpdateJobConfig( 'allowed-updates': mapAllowedUpdatesFromDependabotConfigToJobConfig(update.allow), 'ignore-conditions': mapIgnoreConditionsFromDependabotConfigToJobConfig(update.ignore), 'security-updates-only': update["open-pull-requests-limit"] == 0, - 'security-advisories': [], // TODO: add config option + 'security-advisories': [], // TODO: add config for this! 'source': { 'provider': 'azure', 'api-endpoint': taskInputs.apiEndpointUrl, @@ -115,13 +115,13 @@ function buildUpdateJobConfig( 'prefix-development': update["commit-message"]?.["prefix-development"], 'include-scope': update["commit-message"]?.["include"], }, - 'experiments': undefined, // TODO: add config option - 'max-updater-run-time': undefined, // TODO: add config option? + 'experiments': undefined, // TODO: add config for this! + 'max-updater-run-time': undefined, // TODO: add config for this? 'reject-external-code': (update["insecure-external-code-execution"]?.toLocaleLowerCase() == "allow"), - 'repo-private': undefined, // TODO: add config option? - 'repo-contents-path': undefined, // TODO: add config option? - 'requirements-update-strategy': undefined, // TODO: add config option - 'lockfile-only': undefined, // TODO: add config option + 'repo-private': undefined, // TODO: add config for this? + 'repo-contents-path': undefined, // TODO: add config for this? + 'requirements-update-strategy': undefined, // TODO: add config for this! + 'lockfile-only': undefined, // TODO: add config for this! 'vendor-dependencies': update.vendor, 'debug': taskInputs.debug }, @@ -131,18 +131,23 @@ function buildUpdateJobConfig( function mapDependenciesForSecurityUpdate(dependencyList: any[]): string[] { if (!dependencyList || dependencyList.length == 0) { - // No dependency list snapshot exists yet; Attempt to do a security update for all dependencies, but it will probably fail as it is not supported in dependabot-updater yet - // TODO: Find a way to discover vulnerable dependencies for security-only updates without a dependency list snapshot. - // It would be nice if we could run dependabot-cli (e.g. `dependabot --discover-only`), but this is not supported currently. + // This happens when no previous dependency list snapshot exists yet; + // TODO: Find a way to discover dependencies for a first-time security-only update (no existing dependency list snapshot). + // It would be nice if we could use dependabot-cli for this (e.g. `dependabot --discover-only`), but this is not supported currently. + // TODO: Open a issue in dependabot-cli project, ask how we should handle this scenario. warning( "Security updates can only be performed if there is a previous dependency list snapshot available, but there is none as you have not completed a successful update job yet. " + "Dependabot does not currently support discovering vulnerable dependencies during security-only updates and it is likely that this update operation will fail." ); + + // Attempt to do a security update for "all dependencies"; it will probably fail this is not supported in dependabot-updater yet, but it is best we can do... return []; } - return dependencyList.map(dependency => { - return dependency["name"]; - }); + + // Return only dependencies that are vulnerable, ignore the rest + const dependencyNames = dependencyList.map(dependency => dependency["name"]); + const dependencyVulnerabilities = {}; // TODO: getGitHubSecurityAdvisoriesForDependencies(dependencyNames); + return dependencyNames.filter(dependency => dependencyVulnerabilities[dependency]?.length > 0); } function mapGroupsFromDependabotConfigToJobConfig(dependencyGroups: Record): any[] { diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index 284228b2..3cfa01eb 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -15,7 +15,7 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess private readonly prAuthorClient: AzureDevOpsWebApiClient; private readonly prApproverClient: AzureDevOpsWebApiClient; private readonly existingPullRequests: IPullRequestProperties[]; - private readonly taskVariables: ISharedVariables; + private readonly taskInputs: ISharedVariables; // Custom properties used to store dependabot metadata in projects. // https://learn.microsoft.com/en-us/rest/api/azure/devops/core/projects/set-project-properties @@ -30,11 +30,10 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess public static PR_DEFAULT_AUTHOR_NAME = "dependabot[bot]"; constructor(taskInputs: ISharedVariables, prAuthorClient: AzureDevOpsWebApiClient, prApproverClient: AzureDevOpsWebApiClient, existingPullRequests: IPullRequestProperties[]) { - this.taskVariables = taskInputs; + this.taskInputs = taskInputs; this.prAuthorClient = prAuthorClient; this.prApproverClient = prApproverClient; this.existingPullRequests = existingPullRequests; - this.taskVariables = taskInputs; } /** @@ -56,25 +55,31 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess case 'update_dependency_list': - // Store the dependency list snapshot in project properties, for future reference - return this.prAuthorClient.updateProjectProperty( - project, - DependabotOutputProcessor.PROJECT_PROPERTY_NAME_DEPENDENCY_LIST, - function(existingValue: string) { - const repoDependencyLists = JSON.parse(existingValue || '{}'); - repoDependencyLists[repository] = repoDependencyLists[repository] || {}; - repoDependencyLists[repository][update.job["package-manager"]] = { - 'dependencies': data['dependencies'], - 'dependency-files': data['dependency_files'], - 'last-updated': new Date().toISOString() - }; - - return JSON.stringify(repoDependencyLists); - } - ); + // Store the dependency list snapshot in project properties, if configured + if (this.taskInputs.storeDependencyList) + { + console.info(`Storing the dependency list snapshot for project '${project}'...`); + await this.prAuthorClient.updateProjectProperty( + project, + DependabotOutputProcessor.PROJECT_PROPERTY_NAME_DEPENDENCY_LIST, + function(existingValue: string) { + const repoDependencyLists = JSON.parse(existingValue || '{}'); + repoDependencyLists[repository] = repoDependencyLists[repository] || {}; + repoDependencyLists[repository][update.job["package-manager"]] = { + 'dependencies': data['dependencies'], + 'dependency-files': data['dependency_files'], + 'last-updated': new Date().toISOString() + }; + + return JSON.stringify(repoDependencyLists); + } + ); + } + + return true; case 'create_pull_request': - if (this.taskVariables.skipPullRequests) { + if (this.taskInputs.skipPullRequests) { warning(`Skipping pull request creation as 'skipPullRequests' is set to 'true'`); return true; } @@ -100,15 +105,15 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess branch: targetBranch }, author: { - email: this.taskVariables.authorEmail || DependabotOutputProcessor.PR_DEFAULT_AUTHOR_EMAIL, - name: this.taskVariables.authorName || DependabotOutputProcessor.PR_DEFAULT_AUTHOR_NAME + email: this.taskInputs.authorEmail || DependabotOutputProcessor.PR_DEFAULT_AUTHOR_EMAIL, + name: this.taskInputs.authorName || DependabotOutputProcessor.PR_DEFAULT_AUTHOR_NAME }, title: data['pr-title'], description: data['pr-body'], commitMessage: data['commit-message'], - autoComplete: this.taskVariables.setAutoComplete ? { - ignorePolicyConfigIds: this.taskVariables.autoCompleteIgnoreConfigIds, - mergeStrategy: GitPullRequestMergeStrategy[this.taskVariables.mergeStrategy as keyof typeof GitPullRequestMergeStrategy] + autoComplete: this.taskInputs.setAutoComplete ? { + ignorePolicyConfigIds: this.taskInputs.autoCompleteIgnoreConfigIds, + mergeStrategy: GitPullRequestMergeStrategy[this.taskInputs.mergeStrategy as keyof typeof GitPullRequestMergeStrategy] } : undefined, assignees: update.config.assignees, reviewers: update.config.reviewers, @@ -119,7 +124,7 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess }) // Auto-approve the pull request, if required - if (this.taskVariables.autoApprove && this.prApproverClient && newPullRequestId) { + if (this.taskInputs.autoApprove && this.prApproverClient && newPullRequestId) { await this.prApproverClient.approvePullRequest({ project: project, repository: repository, @@ -130,7 +135,7 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess return newPullRequestId !== undefined; case 'update_pull_request': - if (this.taskVariables.skipPullRequests) { + if (this.taskInputs.skipPullRequests) { warning(`Skipping pull request update as 'skipPullRequests' is set to 'true'`); return true; } @@ -148,12 +153,12 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess repository: repository, pullRequestId: pullRequestToUpdate.id, changes: getPullRequestChangedFilesForOutputData(data), - skipIfCommitsFromUsersOtherThan: this.taskVariables.authorEmail || DependabotOutputProcessor.PR_DEFAULT_AUTHOR_EMAIL, + skipIfCommitsFromUsersOtherThan: this.taskInputs.authorEmail || DependabotOutputProcessor.PR_DEFAULT_AUTHOR_EMAIL, skipIfNoConflicts: true, }); // Re-approve the pull request, if required - if (this.taskVariables.autoApprove && this.prApproverClient && pullRequestWasUpdated) { + if (this.taskInputs.autoApprove && this.prApproverClient && pullRequestWasUpdated) { await this.prApproverClient.approvePullRequest({ project: project, repository: repository, @@ -164,7 +169,7 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess return pullRequestWasUpdated; case 'close_pull_request': - if (this.taskVariables.abandonUnwantedPullRequests) { + if (this.taskInputs.abandonUnwantedPullRequests) { warning(`Skipping pull request closure as 'abandonUnwantedPullRequests' is set to 'true'`); return true; } @@ -176,12 +181,15 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess return false; } + // TODO: GitHub Dependabot will close with reason "Superseded by ${new_pull_request_id}" when another PR supersedes it. + // How do we detect this? Do we need to? + // Close the pull request return await this.prAuthorClient.closePullRequest({ project: project, repository: repository, pullRequestId: pullRequestToClose.id, - comment: this.taskVariables.commentPullRequests ? getPullRequestCloseReasonForOutputData(data) : undefined, + comment: this.taskInputs.commentPullRequests ? getPullRequestCloseReasonForOutputData(data) : undefined, deleteSourceBranch: true }); @@ -235,8 +243,9 @@ export function buildPullRequestProperties(packageManager: string, dependencies: ]; } -export function parseDependencyListProperty(dependencyList: string, repository: string, packageManager: string): any { - const repoDependencyLists = JSON.parse(dependencyList || '{}'); +export function parseProjectDependencyListProperty(properties: Record, repository: string, packageManager: string): any { + const dependencyList = properties?.[DependabotOutputProcessor.PROJECT_PROPERTY_NAME_DEPENDENCY_LIST] || '{}'; + const repoDependencyLists = JSON.parse(dependencyList); return repoDependencyLists[repository]?.[packageManager]; } @@ -298,8 +307,6 @@ function getPullRequestCloseReasonForOutputData(data: any): string { case 'dependency_removed': reason = `Looks like ${leadDependencyName} is no longer a dependency`; break; case 'up_to_date': reason = `Looks like ${leadDependencyName} is up-to-date now`; break; case 'update_no_longer_possible': reason = `Looks like ${leadDependencyName} can no longer be updated`; break; - // TODO: ??? => "Looks like these dependencies are updatable in another way, so this is no longer needed" - // TODO: ??? => "Superseded by ${new_pull_request_id}" } if (reason?.length > 0) { reason += ', so this is no longer needed.'; diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJobConfig.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJobConfig.ts index 64bb06a4..cf6d7948 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJobConfig.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJobConfig.ts @@ -33,8 +33,6 @@ export interface IDependabotUpdateJobConfig { 'dependency-name'?: string, 'source'?: string, 'update-types'?: string[], - // TODO: 'updated-at' config is not in the dependabot.yaml docs, but is in the dependabot-cli and dependabot-core models - // https://github.com/dependabot/dependabot-core/blob/dc7b8d2ad152780e847ab597d40a57f13ab86d4f/common/lib/dependabot/pull_request_creator/message_builder.rb#L762 'updated-at'?: string, 'version-requirement'?: string, }[], diff --git a/extension/tasks/dependabot/dependabotV2/utils/getDockerImageTag.ts b/extension/tasks/dependabot/dependabotV2/utils/getDockerImageTag.ts deleted file mode 100644 index ebd81969..00000000 --- a/extension/tasks/dependabot/dependabotV2/utils/getDockerImageTag.ts +++ /dev/null @@ -1,36 +0,0 @@ -import * as tl from 'azure-pipelines-task-lib/task'; -import * as fs from 'fs'; -import * as path from 'path'; - -/** - * Extract the docker image tag from `dockerImageTag` input or the `task.json` file. - * @returns {string} the version - */ -export default function getDockerImageTag(): string { - let dockerImageTag: string | undefined = tl.getInput('dockerImageTag'); - - if (!dockerImageTag) { - tl.debug('Getting dockerImageTag from task.json file. If you want to override, specify the dockerImageTag input'); - - // Ensure we have the file. Otherwise throw a well readable error. - const filePath = path.join(__dirname, '..', 'task.json'); - if (!fs.existsSync(filePath)) { - throw new Error(`task.json could not be found at '${filePath}'`); - } - - // Ensure the file parsed to an object - let obj: any = JSON.parse(fs.readFileSync(filePath, 'utf-8')); - if (obj === null || typeof obj !== 'object') { - throw new Error('Invalid dependabot config object'); - } - - const versionMajor = obj['version']['Major']; - const versionMinor = obj['version']['Minor']; - if (!!!versionMajor || !!!versionMinor) - throw new Error('Major and/or Minor version could not be parsed from the file'); - - dockerImageTag = `${versionMajor}.${versionMinor}`; - } - - return dockerImageTag; -} diff --git a/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts b/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts index e4be4d00..792edc64 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts @@ -3,7 +3,6 @@ import extractHostname from './extractHostname'; import extractOrganization from './extractOrganization'; import extractVirtualDirectory from './extractVirtualDirectory'; import getAzureDevOpsAccessToken from './getAzureDevOpsAccessToken'; -import getDockerImageTag from './getDockerImageTag'; import getGithubAccessToken from './getGithubAccessToken'; export interface ISharedVariables { @@ -40,6 +39,8 @@ export interface ISharedVariables { authorEmail?: string; authorName?: string; + storeDependencyList: boolean; + /** Determines if the pull requests that dependabot creates should have auto complete set */ setAutoComplete: boolean; /** Merge strategies which can be used to complete a pull request */ @@ -53,7 +54,6 @@ export interface ISharedVariables { autoApproveUserToken: string; /** Determines if the execution should fail when an exception occurs */ - failOnException: boolean; excludeRequirementsToUnlock: string; updaterOptions: string; @@ -71,18 +71,6 @@ export interface ISharedVariables { commentPullRequests: boolean; /** Determines whether to abandon unwanted pull requests */ abandonUnwantedPullRequests: boolean; - - /** List of extra environment variables */ - extraEnvironmentVariables: string[]; - - /** Flag used to forward the host ssh socket */ - forwardHostSshSocket: boolean; - - /** Tag of the docker image to be pulled */ - dockerImageTag: string; - - /** Dependabot command to run */ - command: string; } /** @@ -125,12 +113,13 @@ export default function getSharedVariables(): ISharedVariables { let mergeStrategy = tl.getInput('mergeStrategy', true); let autoCompleteIgnoreConfigIds = tl.getDelimitedInput('autoCompleteIgnoreConfigIds', ';', false).map(Number); + let storeDependencyList = tl.getBoolInput('storeDependencyList', false); + // Prepare variables for auto approve let autoApprove: boolean = tl.getBoolInput('autoApprove', false); let autoApproveUserToken: string = tl.getInput('autoApproveUserToken'); // Prepare control flow variables - let failOnException = tl.getBoolInput('failOnException', true); let excludeRequirementsToUnlock = tl.getInput('excludeRequirementsToUnlock') || ''; let updaterOptions = tl.getInput('updaterOptions'); @@ -145,15 +134,6 @@ export default function getSharedVariables(): ISharedVariables { let commentPullRequests: boolean = tl.getBoolInput('commentPullRequests', false); let abandonUnwantedPullRequests: boolean = tl.getBoolInput('abandonUnwantedPullRequests', true); - let extraEnvironmentVariables = tl.getDelimitedInput('extraEnvironmentVariables', ';', false); - - let forwardHostSshSocket: boolean = tl.getBoolInput('forwardHostSshSocket', false); - - // Prepare variables for the docker image to use - let dockerImageTag: string = getDockerImageTag(); - - let command: string = tl.getBoolInput('useUpdateScriptvNext', false) ? 'update_script_vnext' : 'update_script'; - return { organizationUrl: formattedOrganizationUrl, protocol, @@ -174,6 +154,8 @@ export default function getSharedVariables(): ISharedVariables { authorEmail, authorName, + storeDependencyList, + setAutoComplete, mergeStrategy, autoCompleteIgnoreConfigIds, @@ -181,7 +163,6 @@ export default function getSharedVariables(): ISharedVariables { autoApprove, autoApproveUserToken, - failOnException, excludeRequirementsToUnlock, updaterOptions, @@ -193,13 +174,5 @@ export default function getSharedVariables(): ISharedVariables { skipPullRequests, commentPullRequests, abandonUnwantedPullRequests, - - extraEnvironmentVariables, - - forwardHostSshSocket, - - dockerImageTag, - - command, }; } From 9a727060869ee51c011aa4865d838092231f064d Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Tue, 10 Sep 2024 23:36:04 +1200 Subject: [PATCH 29/57] Implement experiments --- extension/README.md | 25 +++++ .../tasks/dependabot/dependabotV2/task.json | 98 +++++++++---------- .../dependabot-cli/DependabotJobBuilder.ts | 2 +- .../interfaces/IDependabotUpdateJobConfig.ts | 2 +- .../dependabotV2/utils/getSharedVariables.ts | 19 ++-- 5 files changed, 86 insertions(+), 60 deletions(-) diff --git a/extension/README.md b/extension/README.md index 6c7e9f8e..289a6224 100644 --- a/extension/README.md +++ b/extension/README.md @@ -88,3 +88,28 @@ steps: ``` Check the logs for the image that is pulled. + +## Experiments +The following Dependabot experiment flags are known to exist. + +### All package ecyosystems +dedup_branch_names +grouped_updates_experimental_rules +grouped_security_updates_disabled +record_ecosystem_versions +record_update_job_unknown_error +dependency_change_validation +add_deprecation_warn_to_pr_message +threaded_metadata + +### Go +tidy +vendor +goprivate + +### NPM and Yarn +enable_pnpm_yarn_dynamic_engine + +### NuGet +nuget_native_analysis +nuget_dependency_solver diff --git a/extension/tasks/dependabot/dependabotV2/task.json b/extension/tasks/dependabot/dependabotV2/task.json index 1fba3900..df680d57 100644 --- a/extension/tasks/dependabot/dependabotV2/task.json +++ b/extension/tasks/dependabot/dependabotV2/task.json @@ -21,14 +21,14 @@ "instanceNameFormat": "Dependabot", "minimumAgentVersion": "3.232.1", "groups": [ - { - "name": "security_updates", - "displayName": "Security advisories and vulnerabilities", - "isExpanded": false - }, { "name": "pull_requests", "displayName": "Pull request options", + "isExpanded": true + }, + { + "name": "security_updates", + "displayName": "Security advisories and vulnerabilities", "isExpanded": false }, { @@ -57,6 +57,7 @@ "required": false, "helpMarkDown": "When set to `true`, the discovered dependencies will be stored against the project. Defaults to `false`." }, + { "name": "skipPullRequests", "type": "boolean", @@ -66,15 +67,6 @@ "required": false, "helpMarkDown": "When set to `true` the logic to update the dependencies is executed but the actual Pull Requests are not created/updated. Defaults to `false`." }, - { - "name": "commentPullRequests", - "type": "boolean", - "groupName": "pull_requests", - "label": "Comment on abandoned pull requests with close reason.", - "defaultValue": false, - "required": false, - "helpMarkDown": "When set to `true` a comment will be added to abandoned pull requests explanating why it was closed. Defaults to `false`." - }, { "name": "abandonUnwantedPullRequests", "type": "boolean", @@ -84,23 +76,15 @@ "required": false, "helpMarkDown": "When set to `true` pull requests that are no longer needed are closed at the tail end of the execution. Defaults to `false`." }, - { - "name": "authorEmail", - "type": "string", - "groupName": "pull_requests", - "label": "Author email address", - "defaultValue": "", - "required": false, - "helpMarkDown": "The email address to use as the git commit author of the pull requests. Defaults to 'noreply@github.com'." - }, { - "name": "authorName", - "type": "string", + "name": "commentPullRequests", + "type": "boolean", "groupName": "pull_requests", - "label": "Author name", - "defaultValue": "", + "label": "Comment on abandoned pull requests with close reason.", + "defaultValue": false, "required": false, - "helpMarkDown": "The name to use as the git commit author of the pull requests. Defaults to 'dependabot[bot]'." + "helpMarkDown": "When set to `true` a comment will be added to abandoned pull requests explanating why it was closed. Defaults to `false`.", + "visibleRule": "abandonUnwantedPullRequests=true" }, { "name": "setAutoComplete", @@ -115,7 +99,7 @@ "name": "mergeStrategy", "type": "pickList", "groupName": "pull_requests", - "label": "Merge Strategy", + "label": "Auto-complete merge Strategy", "defaultValue": "squash", "required": true, "helpMarkDown": "The merge strategy to use. Learn more [here](https://learn.microsoft.com/en-us/rest/api/azure/devops/git/pull-requests/update?view=azure-devops-rest-5.1&tabs=HTTP#gitpullrequestmergestrategy).", @@ -156,24 +140,25 @@ "helpMarkDown": "A personal access token of the user of that shall be used to approve the created PR automatically. If the same user that creates the PR should approve, this can be left empty. This won't work with if the Build Service with the build service account!", "visibleRule": "autoApprove=true" }, - { - "name": "gitHubConnection", - "type": "connectedService:github:OAuth,PersonalAccessToken,InstallationToken,Token", - "groupName": "github", - "label": "GitHub connection (OAuth or PAT)", + { + "name": "authorEmail", + "type": "string", + "groupName": "pull_requests", + "label": "Git commit uthor email address", "defaultValue": "", "required": false, - "helpMarkDown": "Specify the name of the GitHub service connection to use to connect to the GitHub repositories. The connection must be based on a GitHub user's OAuth or a GitHub personal access token. Learn more about service connections [here](https://aka.ms/AA3am5s)." + "helpMarkDown": "The email address to use as the git commit author of the pull requests. Defaults to 'noreply@github.com'." }, { - "name": "gitHubAccessToken", + "name": "authorName", "type": "string", - "groupName": "github", - "label": "GitHub Personal Access Token.", + "groupName": "pull_requests", + "label": "Git commit author name", "defaultValue": "", "required": false, - "helpMarkDown": "The raw Personal Access Token for accessing GitHub repositories. Use this in place of `gitHubConnection` such as when it is not possible to create a service connection." + "helpMarkDown": "The name to use as the git commit author of the pull requests. Defaults to 'dependabot[bot]'." }, + { "name": "securityAdvisoriesFile", "type": "string", @@ -182,6 +167,7 @@ "helpMarkDown": "The file containing security advisories.", "required": false }, + { "name": "azureDevOpsServiceConnection", "type": "connectedService:Externaltfs", @@ -198,6 +184,26 @@ "required": false, "helpMarkDown": "The Personal Access Token for accessing Azure DevOps repositories. Supply a value here to avoid using permissions for the Build Service either because you cannot change its permissions or because you prefer that the Pull Requests be done by a different user. Use this in place of `azureDevOpsServiceConnection` such as when it is not possible to create a service connection." }, + + { + "name": "gitHubConnection", + "type": "connectedService:github:OAuth,PersonalAccessToken,InstallationToken,Token", + "groupName": "github", + "label": "GitHub connection (OAuth or PAT)", + "defaultValue": "", + "required": false, + "helpMarkDown": "Specify the name of the GitHub service connection to use to connect to the GitHub repositories. The connection must be based on a GitHub user's OAuth or a GitHub personal access token. Learn more about service connections [here](https://aka.ms/AA3am5s)." + }, + { + "name": "gitHubAccessToken", + "type": "string", + "groupName": "github", + "label": "GitHub Personal Access Token.", + "defaultValue": "", + "required": false, + "helpMarkDown": "The raw Personal Access Token for accessing GitHub repositories. Use this in place of `gitHubConnection` such as when it is not possible to create a service connection." + }, + { "name": "targetRepositoryName", "type": "string", @@ -216,20 +222,12 @@ "helpMarkDown": "A semicolon (`;`) delimited list of update identifiers run. Index are zero-based and in the order written in the configuration file. When not present, all the updates are run. This is meant to be used in scenarios where you want to run updates a different times from the same configuration file given you cannot schedule them independently in the pipeline." }, { - "name": "updaterOptions", - "type": "string", - "groupName": "advanced", - "label": "Comma separated list of Dependabot experiments (updater options).", - "required": false, - "helpMarkDown": "Set a list of Dependabot experiments (updater options) in CSV format. Available options depend on the ecosystem. Example: `goprivate=true,kubernetes_updates=true`." - }, - { - "name": "excludeRequirementsToUnlock", + "name": "experiments", "type": "string", "groupName": "advanced", - "label": "Space-separated list of dependency updates requirements to be excluded.", + "label": "Dependabot updater experiments", "required": false, - "helpMarkDown": "Exclude certain dependency updates requirements. See list of allowed values [here](https://github.com/dependabot/dependabot-core/issues/600#issuecomment-407808103). Useful if you have lots of dependencies and the update script too slow. The values provided are space-separated. Example: `own all` to only use the `none` version requirement." + "helpMarkDown": "Comma-seperated list of Dependabot experiments. Available options depend on the ecosystem." } ], "dataSourceBindings": [], diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts index 43b01c7c..660d576a 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts @@ -115,7 +115,7 @@ function buildUpdateJobConfig( 'prefix-development': update["commit-message"]?.["prefix-development"], 'include-scope': update["commit-message"]?.["include"], }, - 'experiments': undefined, // TODO: add config for this! + 'experiments': taskInputs.experiments, 'max-updater-run-time': undefined, // TODO: add config for this? 'reject-external-code': (update["insecure-external-code-execution"]?.toLocaleLowerCase() == "allow"), 'repo-private': undefined, // TODO: add config for this? diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJobConfig.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJobConfig.ts index cf6d7948..4f2fbe00 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJobConfig.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJobConfig.ts @@ -76,7 +76,7 @@ export interface IDependabotUpdateJobConfig { 'prefix-development'?: string, 'include-scope'?: string, }, - 'experiments'?: any, + 'experiments'?: Record, 'max-updater-run-time'?: number, 'reject-external-code'?: boolean, 'repo-private'?: boolean, diff --git a/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts b/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts index 792edc64..dd04716a 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts @@ -53,9 +53,7 @@ export interface ISharedVariables { /** A personal access token of the user that should approve the PR */ autoApproveUserToken: string; - /** Determines if the execution should fail when an exception occurs */ - excludeRequirementsToUnlock: string; - updaterOptions: string; + experiments: Record; /** Determines if verbose log messages are logged */ debug: boolean; @@ -119,9 +117,15 @@ export default function getSharedVariables(): ISharedVariables { let autoApprove: boolean = tl.getBoolInput('autoApprove', false); let autoApproveUserToken: string = tl.getInput('autoApproveUserToken'); - // Prepare control flow variables - let excludeRequirementsToUnlock = tl.getInput('excludeRequirementsToUnlock') || ''; - let updaterOptions = tl.getInput('updaterOptions'); + // Convert experiments from comma separated key vaule pairs to a record + let experiments = tl.getInput('experiments', false)?.split(',')?.reduce( + (acc, cur) => { + let [key, value] = cur.split('=', 2); + acc[key] = value || true; + return acc; + }, + {} as Record + ); let debug: boolean = tl.getVariable('System.Debug')?.localeCompare('true') === 0; @@ -163,8 +167,7 @@ export default function getSharedVariables(): ISharedVariables { autoApprove, autoApproveUserToken, - excludeRequirementsToUnlock, - updaterOptions, + experiments, debug, From 39704b8dde8cfd149aa148d50620d657b4f886f7 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Wed, 11 Sep 2024 00:00:36 +1200 Subject: [PATCH 30/57] Implement requirements-update-strategy and lockfile-only configs --- .../utils/dependabot-cli/DependabotCli.ts | 4 ++-- .../dependabot-cli/DependabotJobBuilder.ts | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts index 196625b5..48a1d490 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts @@ -78,7 +78,7 @@ export class DependabotCli { // Run dependabot update if (!fs.existsSync(jobOutputPath) || fs.statSync(jobOutputPath)?.size == 0) { - console.info(`Running dependabot update job '${jobInputPath}'...`); + console.info(`Running Dependabot update job from '${jobInputPath}'...`); const dependabotTool = tool(which("dependabot", true)).arg(dependabotArguments); const dependabotResultCode = await dependabotTool.execAsync({ silent: !this.debug, @@ -95,7 +95,7 @@ export class DependabotCli { if (fs.existsSync(jobOutputPath)) { const jobOutputs = readJobScenarioOutputFile(jobOutputPath); if (jobOutputs?.length > 0) { - console.info("Processing Dependabot outputs..."); + console.info(`Processing Dependabot outputs from '${jobInputPath}'...`); for (const output of jobOutputs) { // Documentation on the scenario model can be found here: // https://github.com/dependabot/cli/blob/main/internal/model/scenario.go diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts index 660d576a..779730cb 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts @@ -120,8 +120,8 @@ function buildUpdateJobConfig( 'reject-external-code': (update["insecure-external-code-execution"]?.toLocaleLowerCase() == "allow"), 'repo-private': undefined, // TODO: add config for this? 'repo-contents-path': undefined, // TODO: add config for this? - 'requirements-update-strategy': undefined, // TODO: add config for this! - 'lockfile-only': undefined, // TODO: add config for this! + 'requirements-update-strategy': mapVersionStrategyToRequirementsUpdateStrategy(update["versioning-strategy"]), + 'lockfile-only': update["versioning-strategy"] === 'lockfile-only', 'vendor-dependencies': update.vendor, 'debug': taskInputs.debug }, @@ -199,6 +199,20 @@ function mapIgnoreConditionsFromDependabotConfigToJobConfig(ignoreConditions: ID }); } +function mapVersionStrategyToRequirementsUpdateStrategy(versioningStrategy: string): string | undefined { + if (!versioningStrategy) { + return undefined; + } + switch(versioningStrategy) { + case 'auto': return undefined; + case 'increase': return 'bump_versions'; + case 'increase-if-necessary': return 'bump_versions_if_necessary'; + case 'lockfile-only': return 'lockfile_only'; + case 'widen': return 'widen_ranges'; + default: throw new Error(`Invalid dependabot.yaml versioning strategy option '${versioningStrategy}'`); + } +} + function mapRegistryCredentialsFromDependabotConfigToJobConfig(taskInputs: ISharedVariables, registries: Record): any[] { let registryCredentials = new Array(); if (taskInputs.systemAccessToken) { From 612df86450e32700e3727a24f16353d3026c2bff Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Wed, 11 Sep 2024 00:22:39 +1200 Subject: [PATCH 31/57] Fix typo --- .../tasks/dependabot/dependabotV2/utils/getSharedVariables.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts b/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts index dd04716a..af268204 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts @@ -117,7 +117,7 @@ export default function getSharedVariables(): ISharedVariables { let autoApprove: boolean = tl.getBoolInput('autoApprove', false); let autoApproveUserToken: string = tl.getInput('autoApproveUserToken'); - // Convert experiments from comma separated key vaule pairs to a record + // Convert experiments from comma separated key value pairs to a record let experiments = tl.getInput('experiments', false)?.split(',')?.reduce( (acc, cur) => { let [key, value] = cur.split('=', 2); From 3231240d3b570aa294f92d92e320440cac1240ef Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Fri, 13 Sep 2024 22:10:32 +1200 Subject: [PATCH 32/57] Implement PR reviewers, work item references, and labels --- .../tasks/dependabot/dependabotV2/index.ts | 4 +- .../azure-devops/AzureDevOpsWebApiClient.ts | 90 ++++++++++++++++--- .../azure-devops/interfaces/IPullRequest.ts | 2 +- .../resolveAzureDevOpsIdentities.ts | 2 +- .../DependabotOutputProcessor.ts | 2 +- 5 files changed, 83 insertions(+), 17 deletions(-) rename extension/tasks/dependabot/dependabotV2/utils/{ => azure-devops}/resolveAzureDevOpsIdentities.ts (98%) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index d668f4fe..578c929b 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -36,8 +36,8 @@ async function run() { const prApproverClient = taskInputs.autoApprove ? new AzureDevOpsWebApiClient(taskInputs.organizationUrl.toString(), taskInputs.autoApproveUserToken || taskInputs.systemAccessToken) : null; // Fetch the active pull requests created by the author user - const prAuthorActivePullRequests = await prAuthorClient.getMyActivePullRequestProperties( - taskInputs.project, taskInputs.repository + const prAuthorActivePullRequests = await prAuthorClient.getActivePullRequestProperties( + taskInputs.project, taskInputs.repository, await prAuthorClient.getUserId() ); // Initialise the Dependabot updater diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts index 8a35af39..2118d45d 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts @@ -1,27 +1,51 @@ import { debug, warning, error } from "azure-pipelines-task-lib/task" import { WebApi, getPersonalAccessTokenHandler } from "azure-devops-node-api"; -import { CommentThreadStatus, CommentType, ItemContentType, PullRequestAsyncStatus, PullRequestStatus } from "azure-devops-node-api/interfaces/GitInterfaces"; +import { CommentThreadStatus, CommentType, IdentityRefWithVote, ItemContentType, PullRequestAsyncStatus, PullRequestStatus } from "azure-devops-node-api/interfaces/GitInterfaces"; import { IPullRequestProperties } from "./interfaces/IPullRequestProperties"; import { IPullRequest } from "./interfaces/IPullRequest"; import { IFileChange } from "./interfaces/IFileChange"; +import { resolveAzureDevOpsIdentities } from "./resolveAzureDevOpsIdentities"; /** * Wrapper for DevOps WebApi client with helper methods for easier management of dependabot pull requests */ export class AzureDevOpsWebApiClient { + private readonly organisationApiUrl: string; + private readonly accessToken: string; private readonly connection: WebApi; - private userId: string | null = null; + private cachedUserIds: Record; constructor(organisationApiUrl: string, accessToken: string) { + this.organisationApiUrl = organisationApiUrl; + this.accessToken = accessToken; this.connection = new WebApi( organisationApiUrl, getPersonalAccessTokenHandler(accessToken) ); } - private async getUserId(): Promise { - return (this.userId ||= (await this.connection.connect()).authenticatedUser?.id || ""); + /** + * Get the identity of a user by email address. If no email is provided, the identity of the authenticated user is returned. + * @param email + * @returns + */ + public async getUserId(email?: string): Promise { + + // If no email is provided, resolve to the authenticated user + if (!email) { + this.cachedUserIds[this.accessToken] ||= (await this.connection.connect()).authenticatedUser?.id || ""; + return this.cachedUserIds[this.accessToken]; + } + + // Otherwise, do a cached identity lookup of the supplied email address + // TODO: When azure-devops-node-api supports Graph API, use that instead of the REST API + else if (!this.cachedUserIds[email]) { + const identities = await resolveAzureDevOpsIdentities(new URL(this.organisationApiUrl), [email]); + identities.forEach(i => this.cachedUserIds[i.input] ||= i.id); + } + + return this.cachedUserIds[email]; } /** @@ -47,20 +71,20 @@ export class AzureDevOpsWebApiClient { } /** - * Get the properties for all active pull request created by the current user + * Get the properties for all active pull request created by the supplied user * @param project - * @param repository + * @param repository + * @param creator * @returns */ - public async getMyActivePullRequestProperties(project: string, repository: string): Promise { - console.info(`Fetching active pull request properties in '${project}/${repository}'...`); + public async getActivePullRequestProperties(project: string, repository: string, creator: string): Promise { + console.info(`Fetching active pull request properties in '${project}/${repository}' for '${creator}'...`); try { - const userId = await this.getUserId(); const git = await this.connection.getGitApi(); const pullRequests = await git.getPullRequests( repository, { - creatorId: userId, + creatorId: isGuid(creator) ? creator : await this.getUserId(creator), status: PullRequestStatus.Active }, project @@ -131,6 +155,39 @@ export class AzureDevOpsWebApiClient { pr.project ); + // Build the list of the pull request reviewers + // NOTE: Azure DevOps does not have a concept of assignees, only reviewers. + // We treat assignees as required reviewers and all other reviewers as optional. + const allReviewers: IdentityRefWithVote[] = []; + if (pr.assignees?.length > 0) { + for (const assignee of pr.assignees) { + const identityId = isGuid(assignee) ? assignee : await this.getUserId(assignee); + if (identityId) { + allReviewers.push({ + id: identityId, + isRequired: true, + isFlagged: true, + }); + } + else { + warning(` - Unable to resolve assignee identity '${assignee}'`); + } + } + } + if (pr.reviewers?.length > 0) { + for (const reviewer of pr.reviewers) { + const identityId = isGuid(reviewer) ? reviewer : await this.getUserId(reviewer); + if (identityId) { + allReviewers.push({ + id: identityId, + }); + } + else { + warning(` - Unable to resolve reviewer identity '${reviewer}'`); + } + } + } + // Create the pull request console.info(` - Creating pull request to merge '${pr.source.branch}' into '${pr.target.branch}'...`); const pullRequest = await git.createPullRequest( @@ -138,7 +195,11 @@ export class AzureDevOpsWebApiClient { sourceRefName: `refs/heads/${pr.source.branch}`, targetRefName: `refs/heads/${pr.target.branch}`, title: pr.title, - description: pr.description + description: pr.description, + reviewers: allReviewers, + workItemRefs: pr.workItems?.map(id => { return { id: id }; }), + labels: pr.labels?.map(label => { return { name: label }; }), + isDraft: false // TODO: Add config for this? }, pr.repository, pr.project, @@ -473,4 +534,9 @@ function mergeCommitMessage(id: number, title: string, description: string): str // https://developercommunity.visualstudio.com/t/raise-the-character-limit-for-pull-request-descrip/365708#T-N424531 // return `Merged PR ${id}: ${title}\n\n${description}`.slice(0, 3500); -} \ No newline at end of file +} + +function isGuid(guid: string): boolean { + const regex = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/; + return regex.test(guid); +} diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts index 99d5e6c4..11f00c12 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/interfaces/IPullRequest.ts @@ -28,7 +28,7 @@ export interface IPullRequest { assignees?: string[], reviewers?: string[], labels?: string[], - workItems?: number[], + workItems?: string[], changes: IFileChange[], properties?: { name: string, diff --git a/extension/tasks/dependabot/dependabotV2/utils/resolveAzureDevOpsIdentities.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/resolveAzureDevOpsIdentities.ts similarity index 98% rename from extension/tasks/dependabot/dependabotV2/utils/resolveAzureDevOpsIdentities.ts rename to extension/tasks/dependabot/dependabotV2/utils/azure-devops/resolveAzureDevOpsIdentities.ts index 08f03141..77e0a617 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/resolveAzureDevOpsIdentities.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/resolveAzureDevOpsIdentities.ts @@ -1,6 +1,6 @@ import axios from 'axios'; import * as tl from 'azure-pipelines-task-lib/task'; -import extractOrganization from './extractOrganization'; +import extractOrganization from '../extractOrganization'; export interface IIdentity { /** diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index 3cfa01eb..5560c602 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -118,7 +118,7 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess assignees: update.config.assignees, reviewers: update.config.reviewers, labels: update.config.labels?.split(',').map((label) => label.trim()) || [], - workItems: update.config.milestone ? [Number(update.config.milestone)] : [], + workItems: update.config.milestone ? [update.config.milestone] : [], changes: getPullRequestChangedFilesForOutputData(data), properties: buildPullRequestProperties(update.job["package-manager"], dependencies) }) From 15b5799b15aed271ef5aff4562014b30d41b103a Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Fri, 13 Sep 2024 22:12:47 +1200 Subject: [PATCH 33/57] Add start commands for each task version, use V2 by default --- extension/package.json | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/extension/package.json b/extension/package.json index 0e3c216d..0514f255 100644 --- a/extension/package.json +++ b/extension/package.json @@ -6,9 +6,11 @@ "scripts": { "postdependencies": "cp -r node_modules tasks/dependabot/dependabotV1/node_modules && cp -r node_modules tasks/dependabot/dependabotV2/node_modules", "build": "tsc -p .", - "package": "npx tfx-cli extension create --json5", - "start": "node tasks/dependabot/dependabotV2/index.js", - "test": "jest" + "start": "npm run start:V2", + "start:V1": "node tasks/dependabot/dependabotV1/index.js", + "start:V2": "node tasks/dependabot/dependabotV2/index.js", + "test": "jest", + "package": "npx tfx-cli extension create --json5" }, "repository": { "type": "git", From c230fee2e371c9133fbcdc5d9d5a0431efc89711 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Tue, 17 Sep 2024 15:34:01 +1200 Subject: [PATCH 34/57] Update V1 task.json version numbers when publishing the extension --- .github/workflows/extension.yml | 3 +++ extension/tasks/dependabot/dependabotV1/task.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/extension.yml b/.github/workflows/extension.yml index 40791478..5e1006dc 100644 --- a/.github/workflows/extension.yml +++ b/.github/workflows/extension.yml @@ -76,6 +76,9 @@ jobs: - name: Update version numbers in task.json run: | + echo "`jq '.version.Major=1' extension/tasks/dependabot/dependabotV1/task.json`" > extension/tasks/dependabot/dependabotV1/task.json + echo "`jq '.version.Minor=34' extension/tasks/dependabot/dependabotV1/task.json`" > extension/tasks/dependabot/dependabotV1/task.json + echo "`jq '.version.Patch=${{ github.run_number }}' extension/tasks/dependabot/dependabotV1/task.json`" > extension/tasks/dependabot/dependabotV1/task.json echo "`jq '.version.Major=${{ steps.gitversion.outputs.major }}' extension/tasks/dependabot/dependabotV2/task.json`" > extension/tasks/dependabot/dependabotV2/task.json echo "`jq '.version.Minor=${{ steps.gitversion.outputs.minor }}' extension/tasks/dependabot/dependabotV2/task.json`" > extension/tasks/dependabot/dependabotV2/task.json echo "`jq '.version.Patch=${{ github.run_number }}' extension/tasks/dependabot/dependabotV2/task.json`" > extension/tasks/dependabot/dependabotV2/task.json diff --git a/extension/tasks/dependabot/dependabotV1/task.json b/extension/tasks/dependabot/dependabotV1/task.json index f062ac56..2e616838 100644 --- a/extension/tasks/dependabot/dependabotV1/task.json +++ b/extension/tasks/dependabot/dependabotV1/task.json @@ -14,7 +14,7 @@ "demands": [], "version": { "Major": 1, - "Minor": 33, + "Minor": 0, "Patch": 0 }, "deprecated": true, From bc1d550e1516e2b973a2219d50db6a31c142e193 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Thu, 19 Sep 2024 16:41:03 +1200 Subject: [PATCH 35/57] Update documentation --- README.md | 149 ++++++++++++++---- extension/README.md | 102 ++++++------ .../tasks/dependabot/dependabotV2/task.json | 22 +-- 3 files changed, 171 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index 0fdbf255..84166798 100644 --- a/README.md +++ b/README.md @@ -8,70 +8,99 @@ This repository contains tools for updating dependencies in Azure DevOps reposit In this repository you'll find: -1. Dependabot [updater](./updater) in Ruby. See [docs](./docs/updater.md). -2. Dockerfile and build/image for running the updater via Docker [here](./updater/Dockerfile). -3. Dependabot [server](./server/) in .NET/C#. See [docs](./docs/server.md). -4. Azure DevOps [Extension](https://marketplace.visualstudio.com/items?itemName=tingle-software.dependabot) and [source](./extension). See [docs](./docs/extension.md). +1. Azure DevOps [Extension](https://marketplace.visualstudio.com/items?itemName=tingle-software.dependabot), [source code](./extension) and [docs](./docs/extension.md). +1. Dependabot Server, [source code](./server/) and [docs](./docs/server.md). +1. Dependabot Updater image, [Dockerfile](./updater/Dockerfile), [source code](./updater/) and [docs](./docs/updater.md). **(Deprecated since v2.0)** -> The hosted version is available to sponsors (most, but not all). It includes hustle free runs where the infrastructure is maintained for you. Much like the GitHub hosted version. Alternatively, you can run and host your own [server](./docs/server.md). Once you sponsor, you can send out an email to an maintainer or wait till they reach out. This is meant to ease the burden until GitHub/Azure/Microsoft can get it working natively (which could also be never) and hopefully for free. +## Table of Contents +- [Getting started](#getting-started) +- [Using a configuration file](#using-a-configuration-file) +- [Configuring private feeds and registries](#configuring-private-feeds-and-registries) +- [Configuring security advisories and known vulnerabilities](#configuring-security-advisories-and-known-vulnerabilities) +- [Configuring experiments](#configuring-experiments) +- [Unsupported features and configurations](#unsupported-features-and-configurations) + * [Extension Task](#extension-task) + + [dependabot@V2](#dependabotv2) + + [dependabot@V1](#dependabotv1) + * [Server](#server) +- [Migration Guide](#migration-guide) +- [Development Guide](#development-guide) +- [Acknowledgements](#acknowledgements) +- [Issues & Comments](#issues-amp-comments) + +## Getting started + +Unlike the GitHub-hosted version, Dependabot must be explicitly enabled in your Azure DevOps organisation. There are two options available: + +- [Azure DevOps Extension](https://marketplace.visualstudio.com/items?itemName=tingle-software.dependabot) - Ideal if you want to get Dependabot running with minimal administrative effort. The extension runs directly inside your existing pipeline agents and doesn't require hosting of any additional services. Because the extension runs in pipelines, this option does not scale well if you have a large number of projects/repositories. + +- [Hosted Server](./docs/server.md) - Ideal if you have a large number of projects/repositories or prefer to run Dependabot as a managed service instead of using pipeline agents. See [why should I use the server?](./docs/server.md#why-should-i-use-the-server) + + > A hosted version is available to sponsors (most, but not all). It includes hassle free runs where the infrastructure is maintained for you. Much like the GitHub hosted version. Alternatively, you can run and host your own [self-hosted server](./docs/server.md). Once you sponsor, you can send out an email to an maintainer or wait till they reach out. This is meant to ease the burden until GitHub/Azure/Microsoft can get it working natively (which could also be never) and hopefully for free. ## Using a configuration file -Similar to the GitHub native version where you add a `.azuredevops/dependabot.yml` or `.github/dependabot.yml` file, this repository adds support for the same official [configuration options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file) via a file located at `.azuredevops/dependabot.yml` or `.github/dependabot.yml`. This support is only available in the Azure DevOps extension and the [managed version](https://managd.dev). However, the extension does not currently support automatically picking up the file, a pipeline is still required. See [docs](./extension/README.md#usage). +Similar to the GitHub-hosted version, Dependabot is configured using a [dependabot.yml file](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file) located at `.azuredevops/dependabot.yml` or `.github/dependabot.yml` in your repository. -We are well aware that ignore conditions are not explicitly passed and passed on from the extension/server to the container. It is intentional. The ruby script in the docker container does it automatically. If you are having issues, search for related issues such as before creating a new issue. You can also test against various reproductions such as +All [offical configuration options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file) are supported since V2, earlier versions have limited support. See [unsupported features and configurations](#unsupported-features-and-configurations) for more. -## Credentials for private registries and feeds +## Configuring private feeds and registries -Besides accessing the repository only, sometimes private feeds/registries may need to be accessed. -For example a private NuGet feed or a company internal docker registry. +Besides accessing the repository, sometimes private feeds/registries may need to be accessed. For example a private NuGet feed or a company internal docker registry. -Adding configuration options for private registries is setup in `dependabot.yml` -according to the dependabot [description](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#configuration-options-for-private-registries). +Private registries are configured in `dependabot.yml`, refer to the [offical documentation](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#configuration-options-for-private-registries). -Example: +Examples: ```yml version: 2 registries: - my-Extern@Release: - type: nuget-feed - url: https://dev.azure.com/organization1/_packaging/my-Extern@Release/nuget/v3/index.json - token: PAT:${{ MY_DEPENDABOT_ADO_PAT }} + + # Azure DevOps private feed, all views my-analyzers: type: nuget-feed url: https://dev.azure.com/organization2/_packaging/my-analyzers/nuget/v3/index.json - token: PAT:${{ MY_OTHER_PAT }} + token: PAT:${{ MY_DEPENDABOT_ADO_PAT }} + + # Azure DevOps private feed, "Release" view only + my-Extern@Release: + type: nuget-feed + url: https://dev.azure.com/organization1/_packaging/my-Extern@Release/nuget/v3/index.json + token: PAT:${{ MY_DEPENDABOT_ADO_PAT }} + + # Artifactory private feed using PAT artifactory: type: nuget-feed url: https://artifactory.com/api/nuget/v3/myfeed - token: PAT:${{ MY_ARTIFACTORY_PAT }} + token: PAT:${{ MY_DEPENDABOT_ARTIFACTORY_PAT }} + + # Other private feed using basic auth (username/password) telerik: type: nuget-feed url: https://nuget.telerik.com/v3/index.json username: ${{ MY_TELERIK_USERNAME }} password: ${{ MY_TELERIK_PASSWORD }} token: ${{ MY_TELERIK_USERNAME }}:${{ MY_TELERIK_PASSWORD }} + updates: ... ``` -Note: +Notes: 1. `${{ VARIABLE_NAME }}` notation is used liked described [here](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/managing-encrypted-secrets-for-dependabot) -BUT the values will be used from Environment Variables in the pipeline/environment. Template variables are not supported for this replacement. Replacement only works for values considered secret in the registries section i.e. `username`, `password`, `token`, and `key` +BUT the values will be used from pipeline environment variables. Template variables are not supported for this replacement. Replacement only works for values considered secret in the registries section i.e. `username`, `password`, `token`, and `key` -2. When using an Azure DevOps Artifact feed, only the `token` property is required. The token notation should be `PAT:${{ VARIABLE_NAME }}` otherwise the wrong authentication mechanism is used by Dependabot, see [here](https://github.com/tinglesoftware/dependabot-azure-devops/issues/50) for more details. -When working with Azure DevOps Artifacts, some extra permission steps need to be done: +2. When using an Azure DevOps Artifact feed, the token format must be `PAT:${{ VARIABLE_NAME }}` where `VARIABLE_NAME` is a pipeline/environment variable containing the PAT token. The PAT must: - 1. The PAT should have *Packaging Read* permission. - 2. The user owning the PAT must be granted permissions to access the feed either directly or via a group. An easy way for this is to give `Contributor` permissions the `[{project_name}]\Contributors` group under the `Feed Settings -> Permissions` page. The page has the url format: `https://dev.azure.com/{organization}/{project}/_packaging?_a=settings&feed={feed-name}&view=permissions`. + 1. Have `Packaging (Read)` permission. + 2. Be issued by a user with permission to the feed either directly or via a group. An easy way for this is to give `Contributor` permissions the `[{project_name}]\Contributors` group under the `Feed Settings -> Permissions` page. The page has the url format: `https://dev.azure.com/{organization}/{project}/_packaging?_a=settings&feed={feed-name}&view=permissions`. -3. When using a NuGet package server secured with basic auth, the `username`, `password`, and `token` properties are all required. The token notation should be `${{ USERNAME }}:${{ PASSWORD }}`, see [here](https://github.com/tinglesoftware/dependabot-azure-devops/issues/1232#issuecomment-2247616424) for more details. +The following only apply when using the `dependabot@V1` task: -4. When your project contains a `nuget.config` file with custom package source configuration, the `key` property is required for each nuget-feed registry. The key must match between `dependabot.yml` and `nuget.config` otherwise the package source will be duplicated, package source mappings will be ignored, and auth errors will occur during dependency discovery. +3. When using a private NuGet feed secured with basic auth, the `username`, `password`, **and** `token` properties are all required. The token format must be `${{ USERNAME }}:${{ PASSWORD }}`. - If your `nuget.config` looks like this: +4. When your project contains a `nuget.config` file configured with custom package sources, the `key` property is required for each registry. The key must match between `dependabot.yml` and `nuget.config` otherwise the package source will be duplicated, package source mappings will be ignored, and auth errors will occur during dependency discovery. If your `nuget.config` looks like this: ```xml @@ -104,18 +133,68 @@ When working with Azure DevOps Artifacts, some extra permission steps need to be token: PAT:${{ MY_DEPENDABOT_ADO_PAT }} ``` -## Security Advisories, Vulnerabilities, and Updates +## Configuring security advisories and known vulnerabilities + +Security-only updates is a mechanism to only create pull requests for dependencies with vulnerabilities by updating them to the earliest available non-vulnerable version. Security updates are supported in the same way as the GitHub-hosted version provided that a GitHub access token with `public_repo` access is provided in the `gitHubConnection` task input. + +You can provide extra security advisories, such as those for an internal dependency, in a JSON file via the `securityAdvisoriesFile` task input e.g. `securityAdvisoriesFile: '$(Pipeline.Workspace)/advisories.json'`. An example file is available [here](./advisories-example.json). + +## Configuring experiments +Dependabot uses an internal feature flag system called "experiments". Typically, experiments represent new features or changes in logic which are still being tested before becoming generally available. In some cases, you may want to opt-in to experiments to work around known issues or to opt-in to preview features. + +Experiments can be enabled using the `experiments` task input with a comma-seperated list of key/value pairs representing the experiments e.g. `experiments: 'tidy=true,vendor=true,goprivate=*'`. + +The list of experiments is not [publicly] documented, but can be found by searching the dependabot-core repository using queries like ["enabled?(x)"](https://github.com/search?q=repo%3Adependabot%2Fdependabot-core+%2Fenabled%5CW%5C%28.*%5C%29%2F&type=code) and ["fetch(x)"](https://github.com/search?q=repo%3Adependabot%2Fdependabot-core+%2Foptions%5C.fetch%5C%28.*%2C%2F&type=code). The table below details _some_ known experiments as of v0.275.0; this could become out-of-date at anytime. -Security-only updates ia a mechanism to only create pull requests for dependencies with vulnerabilities by updating them to the earliest available non-vulnerable version. Security updates are supported in the same way as the GitHub-hosted version. In addition, you can provide extra advisories, such as those for an internal dependency, in a JSON file via the `securityAdvisoriesFile` input e.g. `securityAdvisoriesFile: '$(Pipeline.Workspace)/advisories.json'`. A file example is available [here](./advisories-example.json). +|Package Ecosystem|Experiment Name|Type|Description| +|--|--|--|--| +| All | dedup_branch_names | true/false | | +| All | grouped_updates_experimental_rules | true/false | | +| All | grouped_security_updates_disabled | true/false | | +| All | record_ecosystem_versions | true/false | | +| All | record_update_job_unknown_error | true/false | | +| All | dependency_change_validation | true/false | | +| All | add_deprecation_warn_to_pr_message | true/false | | +| All | threaded_metadata | true/false | | +| Bundler | bundler_v1_unsupported_error | true/false | | +| Go | tidy | true/false | | +| Go | vendor | true/false | | +| Go | goprivate | string | | +| NPM and Yarn | enable_pnpm_yarn_dynamic_engine | true/false | | +| NuGet | nuget_native_analysis | true/false | | +| NuGet | nuget_dependency_solver | true/false | | -A GitHub access token with `public_repo` access is required to perform the GitHub GraphQL for `securityVulnerabilities`. +## Unsupported features and configurations +We aim to support all offical features and configuration options, but there are some current limitations and exceptions. + +### Extension Task + +#### `dependabot@V2` +- [`schedule` config options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#scheduleinterval) are ignored, use [pipeline scheduled triggers](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/scheduled-triggers?view=azure-devops&tabs=yaml#scheduled-triggers) instead. +- [Security updates only](https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates#overriding-the-default-behavior-with-a-configuration-file) (i.e. `open-pull-requests-limit: 0`) are not supported. _(coming soon)_ + +#### `dependabot@V1` +- [`schedule` config options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#scheduleinterval) are ignored, use [pipeline scheduled triggers](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/scheduled-triggers?view=azure-devops&tabs=yaml#scheduled-triggers) instead. +- [`directories` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#directories) is only supported if task input `useUpdateScriptVNext: true` is set. +- [`groups` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#groups) is only supported if task input `useUpdateScriptVNext: true` is set. +- [`ignore` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#ignore) may not behave to offical specifications unless task input `useUpdateScriptVNext: true` is set. If you are having issues, search for related issues such as before creating a new issue. +- Private feed/registry authentication is known to cause errors with some package ecyosystems. Support is _slightly_ better when task input `useUpdateScriptVNext: true` is set, but not still not fully supported. + + +### Server +- [`directories` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#directories) is not supported. +- [`groups` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#groups) is not supported. + +## Migration Guide +- [Extension Task V1 → V2](./docs/migrations/v1-to-v2) ## Development Guide If you'd like to contribute to the project or just run it locally, view our development guides for: -- [Azure DevOps extension](./docs/extension.md#development-guide) -- [Dependabot updater](./docs/updater.md#development-guide) +- [Azure DevOps Extension](./docs/extension.md#development-guide) +- [Dependabot Server](./docs/server.md#development-guide) +- [Dependabot Updater image](./docs/updater.md#development-guide) **(Deprecated since v2.0)** ## Acknowledgements @@ -129,4 +208,6 @@ The work in this repository is based on inspired and occasionally guided by some ## Issues & Comments -Please leave all comments, bugs, requests, and issues on the Issues page. We'll respond to your request ASAP! +Please leave all issues, bugs, and feature requests on the [issues page](https://github.com/tinglesoftware/dependabot-azure-devops/issues). We'll respond ASAP! + +Use the [discussions page](https://github.com/tinglesoftware/dependabot-azure-devops/discussions) for all other questions and comments. diff --git a/extension/README.md b/extension/README.md index 289a6224..d4b932db 100644 --- a/extension/README.md +++ b/extension/README.md @@ -1,6 +1,6 @@ # Dependabot Azure DevOps Extension -This is the unofficial [dependabot](https://github.com/Dependabot/dependabot-core) extension for [Azure DevOps](https://azure.microsoft.com/en-gb/services/devops/). It will allow you to run Dependabot inside a build pipeline and is accessible [here in the Visual Studio marketplace](https://marketplace.visualstudio.com/items?itemName=tingle-software.dependabot). The extension first has to be installed before you can run it in your pipeline. +This is the unofficial [dependabot](https://github.com/Dependabot/dependabot-core) extension for [Azure DevOps](https://azure.microsoft.com/en-gb/services/devops/). It will allow you to run Dependabot inside a build pipeline. ## Usage @@ -9,7 +9,7 @@ Add a configuration file stored at `.azuredevops/dependabot.yml` or `.github/dep To use in a YAML pipeline: ```yaml -- task: dependabot@1 +- task: dependabot@2 ``` You can schedule the pipeline as is appropriate for your solution. @@ -32,23 +32,58 @@ pool: vmImage: 'ubuntu-latest' # requires macos or ubuntu (windows is not supported) steps: -- task: dependabot@1 +- task: dependabot@2 ``` -This task makes use of a docker image, which may take time to install. Subsequent dependabot tasks in a job will be faster after initially pulling the image using the first task. An alternative way to run your pipelines faster is by leveraging Docker caching in Azure Pipelines (See [#113](https://github.com/tinglesoftware/dependabot-azure-devops/issues/113#issuecomment-894771611)). +## Task Requirements + +The task makes use of [dependabot-cli](https://github.com/dependabot/cli), which requires [Go](https://go.dev/doc/install) (1.22+) and [Docker](https://docs.docker.com/get-started/get-docker/) (with Linux containers) be installed on the pipeline agent. +If you use [Microsoft-hosted agents](https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops&tabs=yaml#software), we recommend using the [ubuntu-latest](https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md) image, which meets all task requirements. + +Dependabot uses Docker containers, which may take time to install if not already cached. Subsequent dependabot tasks in the same job will be faster after initially pulling the images. An alternative way to run your pipelines faster is by leveraging Docker caching in Azure Pipelines (See [#113](https://github.com/tinglesoftware/dependabot-azure-devops/issues/113#issuecomment-894771611)). ## Task Parameters +### `dependabot@V2` **(Preview)** + +|Input|Description| +|--|--| +|skipPullRequests|**_Optional_**. Determines whether to skip creation and updating of pull requests. When set to `true` the logic to update the dependencies is executed but the actual Pull Requests are not created/updated. This is useful for debugging. Defaults to `false`.| +|abandonUnwantedPullRequests|**_Optional_**. Determines whether to abandon unwanted pull requests. Defaults to `false`.| +|commentPullRequests|**_Optional_**. Determines whether to comment on pull requests which an explanation of the reason for closing. Defaults to `false`.| +|setAutoComplete|**_Optional_**. Determines if the pull requests that dependabot creates should have auto complete set. When set to `true`, pull requests that pass all policies will be merged automatically. Defaults to `false`.| +|mergeStrategy|**_Optional_**. The merge strategy to use when auto complete is set. Learn more [here](https://learn.microsoft.com/en-us/rest/api/azure/devops/git/pull-requests/update?view=azure-devops-rest-6.0&tabs=HTTP#gitpullrequestmergestrategy). Defaults to `squash`.| +|autoCompleteIgnoreConfigIds|**_Optional_**. List of any policy configuration Id's which auto-complete should not wait for. Only applies to optional policies. Auto-complete always waits for required (blocking) policies.| +|autoApprove|**_Optional_**. Determines if the pull requests that dependabot creates should be automatically completed. When set to `true`, pull requests will be approved automatically. To use a different user for approval, supply `autoApproveUserToken` input. Defaults to `false`.| +|autoApproveUserToken|**_Optional_**. A personal access token for the user to automatically approve the created PR.| +|authorEmail|**_Optional_**. The email address to use for the change commit author. Can be used to associate the committer with an existing account, to provide a profile picture.| +|authorName|**_Optional_**. The display name to use for the change commit author.| +|securityAdvisoriesFile|**_Optional_**. The path to a JSON file containing additional security advisories to be included when performing package updates. See: [Configuring security advisories and known vulnerabilities](https://github.com/tinglesoftware/dependabot-azure-devops/#configuring-security-advisories-and-known-vulnerabilities).| +|azureDevOpsServiceConnection|**_Optional_**. A Service Connection to use for accessing Azure DevOps. Supply a value here to avoid using permissions for the Build Service either because you cannot change its permissions or because you prefer that the Pull Requests be done by a different user. When not provided, the current authentication scope is used.
See the [documentation](https://learn.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops) to know more about creating a Service Connections| +|azureDevOpsAccessToken|**_Optional_**. The Personal Access Token for accessing Azure DevOps. Supply a value here to avoid using permissions for the Build Service either because you cannot change its permissions or because you prefer that the Pull Requests be done by a different user. When not provided, the current authentication scope is used. In either case, be use the following permissions are granted:
- Code (Full)
- Pull Requests Threads (Read & Write).
See the [documentation](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#create-a-pat) to know more about creating a Personal Access Token.
Use this in place of `azureDevOpsServiceConnection` such as when it is not possible to create a service connection.| +|gitHubConnection|**_Optional_**. The GitHub service connection for authenticating requests against GitHub repositories. This is useful to avoid rate limiting errors. The token must include permissions to read public repositories. See the [GitHub docs](https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token) for more on Personal Access Tokens and [Azure DevOps docs](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#sep-github) for the GitHub service connection.| +|gitHubAccessToken|**_Optional_**. The raw GitHub PAT for authenticating requests against GitHub repositories. Use this in place of `gitHubConnection` such as when it is not possible to create a service connection.| +|storeDependencyList|**_Optional_**. Determines if the last know dependency list information should be stored in the parent DevOps project properties. If enabled, the authenticated user must have the "Project & Team (Write)" permission for the project. Enabling this option improves performance when doing security-only updates. Defaults to `false`.| +|targetRepositoryName|**_Optional_**. The name of the repository to target for processing. If this value is not supplied then the Build Repository Name is used. Supplying this value allows creation of a single pipeline that runs Dependabot against multiple repositories by running a `dependabot` task for each repository to update.| +|targetUpdateIds|**_Optional_**. A semicolon (`;`) delimited list of update identifiers run. Index are zero-based and in the order written in the configuration file. When not present, all the updates are run. This is meant to be used in scenarios where you want to run updates a different times from the same configuration file given you cannot schedule them independently in the pipeline.| +|experiments|**_Optional_**. Comma separated list of Dependabot experiments; available options depend on the ecosystem. Example: `tidy=true,vendor=true,goprivate=*`. See: [Configuring experiments](https://github.com/tinglesoftware/dependabot-azure-devops/#configuring-experiments)| + +### `dependabot@V1` **(Deprecated)** + |Input|Description| |--|--| +|useUpdateScriptvNext|**_Optional_**. Determines if the task should use the new "vNext" update script based on Dependabot Updater (true), or the original update script based on `dry-run.rb` (false). Defaults to `false`. For more information, see: [PR #1186](https://github.com/tinglesoftware/dependabot-azure-devops/pull/1186).| |failOnException|**_Optional_**. Determines if the execution should fail when an exception occurs. Defaults to `true`.| -|updaterOptions|**_Optional_**. Comma separated list of updater options; available options depend on the ecosystem. Example: `goprivate=true,kubernetes_updates=true`.| +|updaterOptions|**_Optional_**. Comma separated list of updater options; available options depend on the ecosystem. Example: `tidy=true,vendor=true,goprivate=*`. See: [Configuring experiments](https://github.com/tinglesoftware/dependabot-azure-devops/#configuring-experiments)| |setAutoComplete|**_Optional_**. Determines if the pull requests that dependabot creates should have auto complete set. When set to `true`, pull requests that pass all policies will be merged automatically. Defaults to `false`.| |mergeStrategy|**_Optional_**. The merge strategy to use when auto complete is set. Learn more [here](https://learn.microsoft.com/en-us/rest/api/azure/devops/git/pull-requests/update?view=azure-devops-rest-6.0&tabs=HTTP#gitpullrequestmergestrategy). Defaults to `squash`.| +|autoCompleteIgnoreConfigIds|**_Optional_**. List of any policy configuration Id's which auto-complete should not wait for. Only applies to optional policies. Auto-complete always waits for required (blocking) policies.| |autoApprove|**_Optional_**. Determines if the pull requests that dependabot creates should be automatically completed. When set to `true`, pull requests will be approved automatically. To use a different user for approval, supply `autoApproveUserToken` input. Defaults to `false`.| |autoApproveUserToken|**_Optional_**. A personal access token for the user to automatically approve the created PR.| |skipPullRequests|**_Optional_**. Determines whether to skip creation and updating of pull requests. When set to `true` the logic to update the dependencies is executed but the actual Pull Requests are not created/updated. This is useful for debugging. Defaults to `false`.| |abandonUnwantedPullRequests|**_Optional_**. Determines whether to abandon unwanted pull requests. Defaults to `false`.| +|commentPullRequests|**_Optional_**. Determines whether to comment on pull requests which an explanation of the reason for closing. Defaults to `false`.| +|securityAdvisoriesFile|**_Optional_**. The path to a JSON file containing additional security advisories to be included when performing package updates. See: [Configuring security advisories and known vulnerabilities](https://github.com/tinglesoftware/dependabot-azure-devops/#configuring-security-advisories-and-known-vulnerabilities).| |gitHubConnection|**_Optional_**. The GitHub service connection for authenticating requests against GitHub repositories. This is useful to avoid rate limiting errors. The token must include permissions to read public repositories. See the [GitHub docs](https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token) for more on Personal Access Tokens and [Azure DevOps docs](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#sep-github) for the GitHub service connection.| |gitHubAccessToken|**_Optional_**. The raw GitHub PAT for authenticating requests against GitHub repositories. Use this in place of `gitHubConnection` such as when it is not possible to create a service connection.| |azureDevOpsServiceConnection|**_Optional_**. A Service Connection to use for accessing Azure DevOps. Supply a value here to avoid using permissions for the Build Service either because you cannot change its permissions or because you prefer that the Pull Requests be done by a different user. When not provided, the current authentication scope is used.
See the [documentation](https://learn.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops) to know more about creating a Service Connections| @@ -61,55 +96,8 @@ This task makes use of a docker image, which may take time to install. Subsequen ## Advanced -In some situations, such as when getting the latest bits for testing, you might want to override the docker image tag that is pulled. Even though doing so is discouraged you can declare a global variable, for example: - -```yaml -trigger: none # Disable CI trigger - -schedules: -- cron: '0 2 * * *' # daily at 2am UTC - always: true # run even when there are no code changes - branches: - include: - - master - batch: true - displayName: Daily - -# variables declared below can be put in one or more Variable Groups for sharing across pipelines -variables: - DEPENDABOT_ALLOW_CONDITIONS: '[{\"dependency-name\":"django*",\"dependency-type\":\"direct\"}]' # packages allowed to be updated - DEPENDABOT_IGNORE_CONDITIONS: '[{\"dependency-name\":"@types/*"}]' # packages ignored to be updated - -pool: - vmImage: 'ubuntu-latest' # requires macos or ubuntu (windows is not supported) - -steps: -- task: dependabot@1 -``` - -Check the logs for the image that is pulled. - -## Experiments -The following Dependabot experiment flags are known to exist. - -### All package ecyosystems -dedup_branch_names -grouped_updates_experimental_rules -grouped_security_updates_disabled -record_ecosystem_versions -record_update_job_unknown_error -dependency_change_validation -add_deprecation_warn_to_pr_message -threaded_metadata - -### Go -tidy -vendor -goprivate - -### NPM and Yarn -enable_pnpm_yarn_dynamic_engine - -### NuGet -nuget_native_analysis -nuget_dependency_solver +- [Configuring private feeds and registries](https://github.com/tinglesoftware/dependabot-azure-devops/#configuring-private-feeds-and-registries) +- [Configuring security advisories and known vulnerabilities](https://github.com/tinglesoftware/dependabot-azure-devops/#configuring-security-advisories-and-known-vulnerabilities) +- [Configuring experiments](https://github.com/tinglesoftware/dependabot-azure-devops/#configuring-experiments) +- [Unsupported features and configurations](https://github.com/tinglesoftware/dependabot-azure-devops/#unsupported-features-and-configurations) +- [Task migration guide for V1 → V2](https://github.com/tinglesoftware/dependabot-azure-devops/blob/main/docs/migrations/v1-to-v2.md) diff --git a/extension/tasks/dependabot/dependabotV2/task.json b/extension/tasks/dependabot/dependabotV2/task.json index df680d57..eaedc6fe 100644 --- a/extension/tasks/dependabot/dependabotV2/task.json +++ b/extension/tasks/dependabot/dependabotV2/task.json @@ -48,16 +48,7 @@ } ], "inputs": [ - { - "name": "storeDependencyList", - "type": "boolean", - "groupName": "advanced", - "label": "Monitor the discovered dependencies", - "defaultValue": false, - "required": false, - "helpMarkDown": "When set to `true`, the discovered dependencies will be stored against the project. Defaults to `false`." - }, - + { "name": "skipPullRequests", "type": "boolean", @@ -203,7 +194,16 @@ "required": false, "helpMarkDown": "The raw Personal Access Token for accessing GitHub repositories. Use this in place of `gitHubConnection` such as when it is not possible to create a service connection." }, - + + { + "name": "storeDependencyList", + "type": "boolean", + "groupName": "advanced", + "label": "Monitor the discovered dependencies", + "defaultValue": false, + "required": false, + "helpMarkDown": "When set to `true`, the discovered dependencies will be stored against the project. Defaults to `false`." + }, { "name": "targetRepositoryName", "type": "string", From bd534fa42161386113cee6bdeb85325c810d9889 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Fri, 20 Sep 2024 00:29:36 +1200 Subject: [PATCH 36/57] Update documentation --- README.md | 8 ++++---- extension/tasks/dependabot/dependabotV1/task.json | 4 ++-- extension/tasks/dependabot/dependabotV2/task.json | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 84166798..d85b07f2 100644 --- a/README.md +++ b/README.md @@ -42,13 +42,13 @@ Unlike the GitHub-hosted version, Dependabot must be explicitly enabled in your Similar to the GitHub-hosted version, Dependabot is configured using a [dependabot.yml file](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file) located at `.azuredevops/dependabot.yml` or `.github/dependabot.yml` in your repository. -All [offical configuration options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file) are supported since V2, earlier versions have limited support. See [unsupported features and configurations](#unsupported-features-and-configurations) for more. +All [official configuration options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file) are supported since V2, earlier versions have limited support. See [unsupported features and configurations](#unsupported-features-and-configurations) for more. ## Configuring private feeds and registries Besides accessing the repository, sometimes private feeds/registries may need to be accessed. For example a private NuGet feed or a company internal docker registry. -Private registries are configured in `dependabot.yml`, refer to the [offical documentation](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#configuration-options-for-private-registries). +Private registries are configured in `dependabot.yml`, refer to the [official documentation](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#configuration-options-for-private-registries). Examples: @@ -165,7 +165,7 @@ The list of experiments is not [publicly] documented, but can be found by search | NuGet | nuget_dependency_solver | true/false | | ## Unsupported features and configurations -We aim to support all offical features and configuration options, but there are some current limitations and exceptions. +We aim to support all official features and configuration options, but there are some current limitations and exceptions. ### Extension Task @@ -177,7 +177,7 @@ We aim to support all offical features and configuration options, but there are - [`schedule` config options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#scheduleinterval) are ignored, use [pipeline scheduled triggers](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/scheduled-triggers?view=azure-devops&tabs=yaml#scheduled-triggers) instead. - [`directories` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#directories) is only supported if task input `useUpdateScriptVNext: true` is set. - [`groups` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#groups) is only supported if task input `useUpdateScriptVNext: true` is set. -- [`ignore` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#ignore) may not behave to offical specifications unless task input `useUpdateScriptVNext: true` is set. If you are having issues, search for related issues such as before creating a new issue. +- [`ignore` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#ignore) may not behave to official specifications unless task input `useUpdateScriptVNext: true` is set. If you are having issues, search for related issues such as before creating a new issue. - Private feed/registry authentication is known to cause errors with some package ecyosystems. Support is _slightly_ better when task input `useUpdateScriptVNext: true` is set, but not still not fully supported. diff --git a/extension/tasks/dependabot/dependabotV1/task.json b/extension/tasks/dependabot/dependabotV1/task.json index 2e616838..82a4bd51 100644 --- a/extension/tasks/dependabot/dependabotV1/task.json +++ b/extension/tasks/dependabot/dependabotV1/task.json @@ -3,7 +3,7 @@ "id": "d98b873d-cf18-41eb-8ff5-234f14697896", "name": "dependabot", "friendlyName": "Dependabot", - "description": "Automatically update dependencies and vulnerabilities in your code", + "description": "Automatically update dependencies and vulnerabilities in your code using [Dependabot Updater](https://github.com/dependabot/dependabot-core/tree/main/updater)", "helpMarkDown": "For help please visit https://github.com/tinglesoftware/dependabot-azure-devops/issues", "helpUrl": "https://github.com/tinglesoftware/dependabot-azure-devops/issues", "releaseNotes": "https://github.com/tinglesoftware/dependabot-azure-devops/releases", @@ -18,7 +18,7 @@ "Patch": 0 }, "deprecated": true, - "deprecationMessage": "The recommended approach to running dependabot has changed since the creation of this task, and is no longer maintained. Please upgrade to the latest version to continue receiving fixes and features. More details: https://github.com/tinglesoftware/dependabot-azure-devops/discussions/1317.", + "deprecationMessage": "This task version is deprecated and is no longer maintained. Please upgrade to the latest version to continue receiving fixes and features. More details: https://github.com/tinglesoftware/dependabot-azure-devops/discussions/1317.", "instanceNameFormat": "Dependabot", "minimumAgentVersion": "3.232.1", "groups": [ diff --git a/extension/tasks/dependabot/dependabotV2/task.json b/extension/tasks/dependabot/dependabotV2/task.json index eaedc6fe..d939576b 100644 --- a/extension/tasks/dependabot/dependabotV2/task.json +++ b/extension/tasks/dependabot/dependabotV2/task.json @@ -3,7 +3,7 @@ "id": "d98b873d-cf18-41eb-8ff5-234f14697896", "name": "dependabot", "friendlyName": "Dependabot", - "description": "Automatically update dependencies and vulnerabilities in your code", + "description": "Automatically update dependencies and vulnerabilities in your code using [Dependabot CLI](https://github.com/dependabot/cli)", "helpMarkDown": "For help please visit https://github.com/tinglesoftware/dependabot-azure-devops/issues", "helpUrl": "https://github.com/tinglesoftware/dependabot-azure-devops/issues", "releaseNotes": "https://github.com/tinglesoftware/dependabot-azure-devops/releases", From 046e929c9bd4971c92f25533273e797c23172537 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Fri, 20 Sep 2024 10:52:13 +1200 Subject: [PATCH 37/57] Update documentation --- README.md | 107 +++++++++++++++++++----------------- docs/extension.md | 83 ++++++++++++++++++++++++---- docs/migrations/v1-to-v2.md | 20 +++++++ docs/server.md | 10 ++-- docs/updater.md | 39 +++++++------ extension/README.md | 2 +- 6 files changed, 181 insertions(+), 80 deletions(-) create mode 100644 docs/migrations/v1-to-v2.md diff --git a/README.md b/README.md index d85b07f2..3752c472 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ In this repository you'll find: * [Extension Task](#extension-task) + [dependabot@V2](#dependabotv2) + [dependabot@V1](#dependabotv1) + * [Updater Docker image](#updater-docker-image) * [Server](#server) - [Migration Guide](#migration-guide) - [Development Guide](#development-guide) @@ -30,12 +31,13 @@ In this repository you'll find: ## Getting started -Unlike the GitHub-hosted version, Dependabot must be explicitly enabled in your Azure DevOps organisation. There are two options available: +Unlike the GitHub-hosted version, Dependabot for Azure DevOps must be explicitly enabled in your organisation. There are two ways to do this: - [Azure DevOps Extension](https://marketplace.visualstudio.com/items?itemName=tingle-software.dependabot) - Ideal if you want to get Dependabot running with minimal administrative effort. The extension runs directly inside your existing pipeline agents and doesn't require hosting of any additional services. Because the extension runs in pipelines, this option does not scale well if you have a large number of projects/repositories. - [Hosted Server](./docs/server.md) - Ideal if you have a large number of projects/repositories or prefer to run Dependabot as a managed service instead of using pipeline agents. See [why should I use the server?](./docs/server.md#why-should-i-use-the-server) + > [!NOTE] > A hosted version is available to sponsors (most, but not all). It includes hassle free runs where the infrastructure is maintained for you. Much like the GitHub hosted version. Alternatively, you can run and host your own [self-hosted server](./docs/server.md). Once you sponsor, you can send out an email to an maintainer or wait till they reach out. This is meant to ease the burden until GitHub/Azure/Microsoft can get it working natively (which could also be never) and hopefully for free. ## Using a configuration file @@ -86,52 +88,51 @@ updates: ... ``` -Notes: +Note when using authentication secrets in configuration files: -1. `${{ VARIABLE_NAME }}` notation is used liked described [here](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/managing-encrypted-secrets-for-dependabot) +> [!NOTE] +> `${{ VARIABLE_NAME }}` notation is used liked described [here](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/managing-encrypted-secrets-for-dependabot) BUT the values will be used from pipeline environment variables. Template variables are not supported for this replacement. Replacement only works for values considered secret in the registries section i.e. `username`, `password`, `token`, and `key` -2. When using an Azure DevOps Artifact feed, the token format must be `PAT:${{ VARIABLE_NAME }}` where `VARIABLE_NAME` is a pipeline/environment variable containing the PAT token. The PAT must: - - 1. Have `Packaging (Read)` permission. - 2. Be issued by a user with permission to the feed either directly or via a group. An easy way for this is to give `Contributor` permissions the `[{project_name}]\Contributors` group under the `Feed Settings -> Permissions` page. The page has the url format: `https://dev.azure.com/{organization}/{project}/_packaging?_a=settings&feed={feed-name}&view=permissions`. - -The following only apply when using the `dependabot@V1` task: - -3. When using a private NuGet feed secured with basic auth, the `username`, `password`, **and** `token` properties are all required. The token format must be `${{ USERNAME }}:${{ PASSWORD }}`. - -4. When your project contains a `nuget.config` file configured with custom package sources, the `key` property is required for each registry. The key must match between `dependabot.yml` and `nuget.config` otherwise the package source will be duplicated, package source mappings will be ignored, and auth errors will occur during dependency discovery. If your `nuget.config` looks like this: - - ```xml - - - - - - - - - - - - - - - - - ``` - - Then your `dependabot.yml` registry should look like this: - - ```yml - version: 2 - registries: - my-org: - type: nuget-feed - key: my-organisation1-nuget - url: https://dev.azure.com/my-organization/_packaging/my-nuget-feed/nuget/v3/index.json - token: PAT:${{ MY_DEPENDABOT_ADO_PAT }} - ``` +> [!NOTE] +> When using an Azure DevOps Artifact feed, the token format must be `PAT:${{ VARIABLE_NAME }}` where `VARIABLE_NAME` is a pipeline/environment variable containing the PAT token. The PAT must: +> 1. Have `Packaging (Read)` permission. +> 2. Be issued by a user with permission to the feed either directly or via a group. An easy way for this is to give `Contributor` permissions the `[{project_name}]\Contributors` group under the `Feed Settings -> Permissions` page. The page has the url format: `https://dev.azure.com/{organization}/{project}/_packaging?_a=settings&feed={feed-name}&view=permissions`. + +> [!NOTE] +> When using a private NuGet feed secured with basic auth, the `username`, `password`, **and** `token` properties are all required. The token format must be `${{ USERNAME }}:${{ PASSWORD }}`. + +> [!NOTE] +> When your project contains a `nuget.config` file configured with custom package sources, the `key` property is required for each registry. The key must match between `dependabot.yml` and `nuget.config` otherwise the package source will be duplicated, package source mappings will be ignored, and auth errors will occur during dependency discovery. If your `nuget.config` looks like this: +> ```xml +> +> +> +> +> +> +> +> +> +> +> +> +> +> +> +> +> ``` +> +> Then your `dependabot.yml` registry should look like this: +> ```yml +> version: 2 +> registries: +> my-org: +> type: nuget-feed +> key: my-organisation1-nuget +> url: https://dev.azure.com/my-organization/_packaging/my-nuget-feed/nuget/v3/index.json +> token: PAT:${{ MY_DEPENDABOT_ADO_PAT }} +> ``` ## Configuring security advisories and known vulnerabilities @@ -140,13 +141,17 @@ Security-only updates is a mechanism to only create pull requests for dependenci You can provide extra security advisories, such as those for an internal dependency, in a JSON file via the `securityAdvisoriesFile` task input e.g. `securityAdvisoriesFile: '$(Pipeline.Workspace)/advisories.json'`. An example file is available [here](./advisories-example.json). ## Configuring experiments -Dependabot uses an internal feature flag system called "experiments". Typically, experiments represent new features or changes in logic which are still being tested before becoming generally available. In some cases, you may want to opt-in to experiments to work around known issues or to opt-in to preview features. +Dependabot uses an internal feature flag system called "experiments". Typically, experiments represent new features or changes in logic which are still being ]internal] tested before becoming generally available. In some cases, you may want to opt-in to experiments to work around known issues or to opt-in to preview features. -Experiments can be enabled using the `experiments` task input with a comma-seperated list of key/value pairs representing the experiments e.g. `experiments: 'tidy=true,vendor=true,goprivate=*'`. +Experiments can be enabled using the `experiments` task input with a comma-seperated list of key/value pairs representing the enabled experiments e.g. `experiments: 'tidy=true,vendor=true,goprivate=*'`. -The list of experiments is not [publicly] documented, but can be found by searching the dependabot-core repository using queries like ["enabled?(x)"](https://github.com/search?q=repo%3Adependabot%2Fdependabot-core+%2Fenabled%5CW%5C%28.*%5C%29%2F&type=code) and ["fetch(x)"](https://github.com/search?q=repo%3Adependabot%2Fdependabot-core+%2Foptions%5C.fetch%5C%28.*%2C%2F&type=code). The table below details _some_ known experiments as of v0.275.0; this could become out-of-date at anytime. +> [!TIP] +> The list of experiments are not [publicly] documented, but can be found by searching the dependabot-core GitHub repository using queries like ["enabled?(x)"](https://github.com/search?q=repo%3Adependabot%2Fdependabot-core+%2Fenabled%5CW%5C%28.*%5C%29%2F&type=code) and ["fetch(x)"](https://github.com/search?q=repo%3Adependabot%2Fdependabot-core+%2Foptions%5C.fetch%5C%28.*%2C%2F&type=code). -|Package Ecosystem|Experiment Name|Type|Description| +> [!NOTE] +> For convenience, the known experiments as of v0.275.0 as listed below; this could become out-of-date at anytime. + +|Package Ecosystem|Experiment Name|Value Type|Description| |--|--|--|--| | All | dedup_branch_names | true/false | | | All | grouped_updates_experimental_rules | true/false | | @@ -178,12 +183,16 @@ We aim to support all official features and configuration options, but there are - [`directories` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#directories) is only supported if task input `useUpdateScriptVNext: true` is set. - [`groups` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#groups) is only supported if task input `useUpdateScriptVNext: true` is set. - [`ignore` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#ignore) may not behave to official specifications unless task input `useUpdateScriptVNext: true` is set. If you are having issues, search for related issues such as before creating a new issue. -- Private feed/registry authentication is known to cause errors with some package ecyosystems. Support is _slightly_ better when task input `useUpdateScriptVNext: true` is set, but not still not fully supported. +- Private feed/registry authentication is known to cause errors with some package ecyosystems. Support is _slightly_ improved when task input `useUpdateScriptVNext: true` is set, but not still not fully supported. See [problems with authentication](./docs/migrations/v1-to-v2.md#resolving-private-feedregistry-authentication-issues) for more. +### Updater Docker image +- Private feed/registry authentication is known to cause errors with some package ecyosystems. See [problems with authentication](./docs/migrations/v1-to-v2.md#resolving-private-feedregistry-authentication-issues) for more. ### Server + - [`directories` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#directories) is not supported. - [`groups` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#groups) is not supported. +- Private feed/registry authentication is known to cause errors with some package ecyosystems. See [problems with authentication](./docs/migrations/v1-to-v2.md#resolving-private-feedregistry-authentication-issues) for more. ## Migration Guide - [Extension Task V1 → V2](./docs/migrations/v1-to-v2) diff --git a/docs/extension.md b/docs/extension.md index 408f0755..247f6f02 100644 --- a/docs/extension.md +++ b/docs/extension.md @@ -3,21 +3,24 @@ - [Using the extension](#using-the-extension) - [Development guide](#development-guide) - - [Getting the development environment ready](#getting-the-development-environment-ready) - - [Building the extension](#building-the-extension) - - [Installing the extension](#installing-the-extension) - - [Running the unit tests](#running-the-unit-tests) + * [Getting the development environment ready](#getting-the-development-environment-ready) + * [Building the extension](#building-the-extension) + * [Installing the extension](#installing-the-extension) + * [Running the task locally](#running-the-task-locally) + * [Running the unit tests](#running-the-unit-tests) +- [Architecture](#architecture) + * [Task V2 high-level update process diagram](#task-v2-high-level-update-process-diagram) + # Using the extension -See the extension [README.md](../extension/README.md). +Refer to the extension [README.md](../extension/README.md). # Development guide ## Getting the development environment ready -First, ensure you have [Node.js](https://docs.docker.com/engine/install/) v18+ installed. -Next, install project dependencies with npm: +Install [Node.js](https://docs.docker.com/engine/install/) v18 or higher; Install project dependencies using NPM: ```bash cd extension @@ -31,7 +34,7 @@ cd extension npm run build ``` -To generate the Azure DevOps `.vsix` extension package for testing, you'll first need to [create a publisher account](https://learn.microsoft.com/en-us/azure/devops/extend/publish/overview?view=azure-devops#create-a-publisher) on the [Visual Studio Marketplace Publishing Portal](https://marketplace.visualstudio.com/manage/createpublisher?managePageRedirect=true). After this, override your publisher ID below and generate the extension with: +To then generate the a Azure DevOps `.vsix` extension package for testing, you'll first need to [create a publisher account](https://learn.microsoft.com/en-us/azure/devops/extend/publish/overview?view=azure-devops#create-a-publisher) for the [Visual Studio Marketplace Publishing Portal](https://marketplace.visualstudio.com/manage/createpublisher?managePageRedirect=true). After this, use `npm run package` to build the package, with an override for your publisher ID: ```bash npm run package -- --overrides-file overrides.local.json --rev-version --publisher your-publisher-id-here @@ -39,17 +42,77 @@ npm run package -- --overrides-file overrides.local.json --rev-version --publish ## Installing the extension -To test the extension in Azure DevOps, you'll first need to build the extension `.vsix` file (see above). After this, [publish your extension](https://learn.microsoft.com/en-us/azure/devops/extend/publish/overview?view=azure-devops#publish-your-extension), then [install your extension](https://learn.microsoft.com/en-us/azure/devops/extend/publish/overview?view=azure-devops#install-your-extension). +To test the extension in a Azure DevOps organisation: +1. [Build the extension `.vsix` package](#building-the-extension) +1. [Publish the extension to your publisher account](https://learn.microsoft.com/en-us/azure/devops/extend/publish/overview?view=azure-devops#publish-your-extension) +1. [Share the extension with the organisation](https://learn.microsoft.com/en-us/azure/devops/extend/publish/overview?view=azure-devops#share-your-extension). ## Running the task locally - +To run the latest task version: ```bash npm start ``` +To run a specific task version: +```bash +npm run start:V1 # runs dependabotV1 task +npm run start:V2 # runs dependabotV2 task +``` ## Running the unit tests ```bash cd extension npm test ``` + +# Architecture + +## Task V2 high-level update process diagram +High-level sequence diagram illustrating how the `dependabotV2` task performs updates using [dependabot-cli](https://github.com/dependabot/cli). See [how dependabot-cli works](https://github.com/dependabot/cli?tab=readme-ov-file#how-it-works) for more details. +```mermaid + sequenceDiagram + participant ext as Dependabot DevOps Extension + participant agent as DevOps Pipeline Agent + participant devops as DevOps API + participant cli as Dependabot CLI + participant core as Dependabot Updater + participant feed as Package Feed + + ext->>ext: Read and parse `dependabot.yml` + ext->>ext: Write `job.yaml` + ext->>agent: Download dependabot-cli from github + ext->>+cli: Execute `dependabot update -f job.yaml -o update-scenario.yaml` + cli->>+core: Run update for `job.yaml` with proxy and dependabot-updater docker containers + core->>devops: Fetch source files from repository + core->>core: Discover dependencies + loop for each dependency + core->>feed: Fetch latest version + core->>core: Update dependency files + end + core-->>-cli: Report outputs + cli->>cli: Write outputs to `update-sceario.yaml` + cli-->>-ext: Update completed + + ext->>ext: Read and parse `update-sceario.yaml` + loop for each output + alt when output is "create_pull_request" + ext->>devops: Create pull request source branch + ext->>devops: Push commit to source branch + ext->>devops: Create pull request + ext->>devops: Set auto-approve + ext->>devops: Set auto-complete + end + alt when output is "update_pull_request" + ext->>devops: Push commit to pull request + ext->>devops: Update pull request description + ext->>devops: Set auto-approve + ext->>devops: Set auto-complete + end + alt when output is "close_pull_request" + ext->>devops: Create comment thread on pull request with close reason + ext->>devops: Abandon pull request + ext->>devops: Delete source branch + end + end + +``` diff --git a/docs/migrations/v1-to-v2.md b/docs/migrations/v1-to-v2.md new file mode 100644 index 00000000..c77feb67 --- /dev/null +++ b/docs/migrations/v1-to-v2.md @@ -0,0 +1,20 @@ + +# Table of Contents +- [Summary of changes V1 → V2](#summary-of-changes-v1-v2) + * [Resolving private feed/registry authentication issues](#resolving-private-feedregistry-authentication-issues) +- [Breaking changes V1 → V2](#breaking-changes-v1-v2) +- [Steps to migrate V1 → V2](#steps-to-migrate-v1-v2) + +# Summary of changes V1 → V2 +... + +See [extension task architecture](../extension.md#architecture) for more technical details. + +## Resolving private feed/registry authentication issues +... + +# Breaking changes V1 → V2 +... + +# Steps to migrate V1 → V2 +... diff --git a/docs/server.md b/docs/server.md index 5aead255..9bd18a49 100644 --- a/docs/server.md +++ b/docs/server.md @@ -4,10 +4,10 @@ - [Why should I use the server?](#why-should-i-use-the-server) - [Composition](#composition) - [Deployment](#deployment) - - [Single click deployment](#single-click-deployment) - - [Deployment Parameters](#deployment-parameters) - - [Deployment with CLI](#deployment-with-cli) - - [Service Hooks and Subscriptions](#service-hooks-and-subscriptions) + * [Single click deployment](#single-click-deployment) + * [Deployment Parameters](#deployment-parameters) + * [Deployment with CLI](#deployment-with-cli) + * [Service Hooks and Subscriptions](#service-hooks-and-subscriptions) - [Keeping updated](#keeping-updated) # Why should I use the server? @@ -59,10 +59,12 @@ The deployment exposes the following parameters that can be tuned to suit the se |githubToken|Access token for authenticating requests to GitHub. Required for vulnerability checks and to avoid rate limiting on free requests|No|<empty>| |imageTag|The image tag to use when pulling the docker containers. A tag also defines the version. You should avoid using `latest`. Example: `1.1.0`|No|<version-downloaded>| +> [!NOTE] > The template includes a User Assigned Managed Identity, which is used when performing Azure Resource Manager operations such as deletions. In the deployment it creates the role assignments that it needs. These role assignments are on the resource group that you deploy to. ## Deployment with CLI +> [!IMPORTANT] > Ensure the Azure CLI tools are installed and that you are logged in. For a one time deployment, it is similar to how you deploy other resources on Azure. diff --git a/docs/updater.md b/docs/updater.md index e27e042b..2e66f76f 100644 --- a/docs/updater.md +++ b/docs/updater.md @@ -1,24 +1,27 @@ +> [!WARNING] +> Use of the Dependabot Updater image is no longer recommended since v2.0; This updater image is considered an internal component within Dependabot and is not intended to be run directly without the use of a credentials proxy. See [unsupported features and configuration](../README.md#unsupported-features-and-configurations) for more details on the limitations of this image. + # Table of Contents - [Running the updater](#running-the-updater) - - [Environment variables](#environment-variables) + * [Environment Variables](#environment-variables) - [Development guide](#development-guide) - - [Getting the development environment ready](#getting-the-development-environment-ready) - - [Building the Docker image](#building-the-docker-image) - - [Running your code changes](#running-your-code-changes) - - [Running the code linter](#running-the-code-linter) - - [Running the unit tests](#running-the-unit-tests) + * [Getting the development environment ready](#getting-the-development-environment-ready) + * [Building the Docker image](#building-the-docker-image) + * [Running your code changes](#running-your-code-changes) + * [Running the code linter](#running-the-code-linter) + * [Running the unit tests](#running-the-unit-tests) # Running the updater -First, you need to pull the docker image locally to your machine: +[Build](#building-the-docker-image) or pull the docker image: ```bash docker pull ghcr.io/tinglesoftware/dependabot-updater- ``` -Next create and run a container from the image. The full list of container options are detailed in [Environment variables](#environment-variables); at minimum the command should be: +Create and run a container based on the image. The full list of container options are detailed in [environment variables](#environment-variables); at minimum the command should be: ```bash docker run --rm -t \ @@ -77,7 +80,7 @@ docker run --rm -t \ ## Environment Variables -To run the script, some environment variables are required. +The following environment variables are required when running the container. |Variable Name|Supported Command(s)|Description| |--|--|--| @@ -135,10 +138,12 @@ To run the script, some environment variables are required. ## Getting the development environment ready -First, ensure you have [Docker](https://docs.docker.com/engine/install/) and [Ruby](https://www.ruby-lang.org/en/documentation/installation/) installed. -On Linux, you'll need the the build essentials and Ruby development packages too; These are typically `build-essentials` and `ruby-dev`. +Install [Docker](https://docs.docker.com/engine/install/) and [Ruby](https://www.ruby-lang.org/en/documentation/installation/). + +> [!TIP] +> If developing in Linux, you'll also need the the build essentials and Ruby development packages; These are typically `build-essentials` and `ruby-dev`. -Next, install project build tools with bundle: +Install the project build tools using Bundle: ```bash cd updater @@ -159,21 +164,23 @@ docker build \ . ``` -In some scenarios, you may want to set `BASE_VERSION` to a specific version instead of "latest". -See [updater/Dockerfile](../updater/Dockerfile) for a more detailed explanation. +> [!TIP] +> In some scenarios, you may want to set `BASE_VERSION` to a specific version instead of "latest". +> See [updater/Dockerfile](../updater/Dockerfile) for a more detailed explanation. ## Running your code changes -To test run your code changes, you'll first need to build the updater Docker image (see above), then run the updater Docker image in a container with all the required environment variables (see above). +To test run your code changes, you'll first need to [build the Docker image](#building-the-docker-image), then run the Docker image in a container with all the [required environment variables](#environment-variables). ## Running the code linter ```bash cd updater bundle exec rubocop -bundle exec rubocop -a # to automatically fix any correctable offenses ``` +> [!TIP] +> To automatically fix correctable linting issues, use `bundle exec rubocop -a` ## Running the unit tests ```bash diff --git a/extension/README.md b/extension/README.md index d4b932db..66f4680a 100644 --- a/extension/README.md +++ b/extension/README.md @@ -44,7 +44,7 @@ Dependabot uses Docker containers, which may take time to install if not already ## Task Parameters -### `dependabot@V2` **(Preview)** +### `dependabot@V2` |Input|Description| |--|--| From dbed1f9344d32a48e10d797c30dbaad5c6ec3a47 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Fri, 20 Sep 2024 12:17:58 +1200 Subject: [PATCH 38/57] Update documentation --- README.md | 52 ++++++++-------- docs/migrations/v1-to-v2.md | 59 ++++++++++++++++--- docs/updater.md | 4 +- extension/README.md | 4 +- .../tasks/dependabot/dependabotV2/task.json | 8 +-- 5 files changed, 86 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 3752c472..74403c55 100644 --- a/README.md +++ b/README.md @@ -31,20 +31,20 @@ In this repository you'll find: ## Getting started -Unlike the GitHub-hosted version, Dependabot for Azure DevOps must be explicitly enabled in your organisation. There are two ways to do this: +Unlike the GitHub-hosted version, Dependabot for Azure DevOps must be explicitly setup in your organisation, creating a `dependabot.yml` file alone will **not** enable updates. There are two ways to enable Dependabot: -- [Azure DevOps Extension](https://marketplace.visualstudio.com/items?itemName=tingle-software.dependabot) - Ideal if you want to get Dependabot running with minimal administrative effort. The extension runs directly inside your existing pipeline agents and doesn't require hosting of any additional services. Because the extension runs in pipelines, this option does not scale well if you have a large number of projects/repositories. +- [Azure DevOps Extension](https://marketplace.visualstudio.com/items?itemName=tingle-software.dependabot) - Ideal if you want to get Dependabot running with minimal administrative effort. The extension can run directly inside your existing pipeline agents and doesn't require hosting of any additional services. Because the extension runs in pipelines, this option does not scale well if you have a large number of projects/repositories. - [Hosted Server](./docs/server.md) - Ideal if you have a large number of projects/repositories or prefer to run Dependabot as a managed service instead of using pipeline agents. See [why should I use the server?](./docs/server.md#why-should-i-use-the-server) - > [!NOTE] - > A hosted version is available to sponsors (most, but not all). It includes hassle free runs where the infrastructure is maintained for you. Much like the GitHub hosted version. Alternatively, you can run and host your own [self-hosted server](./docs/server.md). Once you sponsor, you can send out an email to an maintainer or wait till they reach out. This is meant to ease the burden until GitHub/Azure/Microsoft can get it working natively (which could also be never) and hopefully for free. +> [!NOTE] +> A hosted version is available to sponsors (most, but not all). It includes hassle free runs where the infrastructure is maintained for you. Much like the GitHub hosted version. Alternatively, you can run and host your own [self-hosted server](./docs/server.md). Once you sponsor, you can send out an email to an maintainer or wait till they reach out. This is meant to ease the burden until GitHub/Azure/Microsoft can get it working natively (which could also be never) and hopefully for free. ## Using a configuration file Similar to the GitHub-hosted version, Dependabot is configured using a [dependabot.yml file](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file) located at `.azuredevops/dependabot.yml` or `.github/dependabot.yml` in your repository. -All [official configuration options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file) are supported since V2, earlier versions have limited support. See [unsupported features and configurations](#unsupported-features-and-configurations) for more. +Most [official configuration options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file) are supported since V2; Earlier versions have serveral limitations, see [unsupported features and configurations](#unsupported-features-and-configurations) for more. ## Configuring private feeds and registries @@ -90,20 +90,20 @@ updates: Note when using authentication secrets in configuration files: -> [!NOTE] +> [!IMPORTANT] > `${{ VARIABLE_NAME }}` notation is used liked described [here](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/managing-encrypted-secrets-for-dependabot) BUT the values will be used from pipeline environment variables. Template variables are not supported for this replacement. Replacement only works for values considered secret in the registries section i.e. `username`, `password`, `token`, and `key` -> [!NOTE] +> [!IMPORTANT] > When using an Azure DevOps Artifact feed, the token format must be `PAT:${{ VARIABLE_NAME }}` where `VARIABLE_NAME` is a pipeline/environment variable containing the PAT token. The PAT must: > 1. Have `Packaging (Read)` permission. > 2. Be issued by a user with permission to the feed either directly or via a group. An easy way for this is to give `Contributor` permissions the `[{project_name}]\Contributors` group under the `Feed Settings -> Permissions` page. The page has the url format: `https://dev.azure.com/{organization}/{project}/_packaging?_a=settings&feed={feed-name}&view=permissions`. > [!NOTE] -> When using a private NuGet feed secured with basic auth, the `username`, `password`, **and** `token` properties are all required. The token format must be `${{ USERNAME }}:${{ PASSWORD }}`. +> When using `dependabot@V1` with a private feed/registry secured with basic auth, the `username`, `password`, **and** `token` properties are all required. The token format must be `${{ USERNAME }}:${{ PASSWORD }}`. > [!NOTE] -> When your project contains a `nuget.config` file configured with custom package sources, the `key` property is required for each registry. The key must match between `dependabot.yml` and `nuget.config` otherwise the package source will be duplicated, package source mappings will be ignored, and auth errors will occur during dependency discovery. If your `nuget.config` looks like this: +> When using `dependabot@V1` with a repository containing a `nuget.config` file configured with custom package sources, the `key` property is required for each registry. The key must match between `dependabot.yml` and `nuget.config` otherwise the package source will be duplicated, package source mappings will be ignored, and auth errors will occur during dependency discovery. If your `nuget.config` looks like this: > ```xml > > @@ -143,13 +143,13 @@ You can provide extra security advisories, such as those for an internal depende ## Configuring experiments Dependabot uses an internal feature flag system called "experiments". Typically, experiments represent new features or changes in logic which are still being ]internal] tested before becoming generally available. In some cases, you may want to opt-in to experiments to work around known issues or to opt-in to preview features. -Experiments can be enabled using the `experiments` task input with a comma-seperated list of key/value pairs representing the enabled experiments e.g. `experiments: 'tidy=true,vendor=true,goprivate=*'`. +Experiments vary depending on the package ecyosystem used; They can be enabled using the `experiments` task input with a comma-seperated list of key/value pairs representing the experiments e.g. `experiments: 'tidy=true,vendor=true,goprivate=*'`. -> [!TIP] -> The list of experiments are not [publicly] documented, but can be found by searching the dependabot-core GitHub repository using queries like ["enabled?(x)"](https://github.com/search?q=repo%3Adependabot%2Fdependabot-core+%2Fenabled%5CW%5C%28.*%5C%29%2F&type=code) and ["fetch(x)"](https://github.com/search?q=repo%3Adependabot%2Fdependabot-core+%2Foptions%5C.fetch%5C%28.*%2C%2F&type=code). +> [!WARNING] +> For convenience, known experiments as of v0.275.0 are listed below; **This be out-of-date at the time of reading.** -> [!NOTE] -> For convenience, the known experiments as of v0.275.0 as listed below; this could become out-of-date at anytime. +> [!TIP] +> To find the latest list of Dependabot experiments, search the `dependabot-core` GitHub repository using queries like ["enabled?(x)"](https://github.com/search?q=repo%3Adependabot%2Fdependabot-core+%2Fenabled%5CW%5C%28.*%5C%29%2F&type=code) and ["options.fetch(x)"](https://github.com/search?q=repo%3Adependabot%2Fdependabot-core+%2Foptions%5C.fetch%5C%28.*%2C%2F&type=code). |Package Ecosystem|Experiment Name|Value Type|Description| |--|--|--|--| @@ -170,29 +170,29 @@ Experiments can be enabled using the `experiments` task input with a comma-seper | NuGet | nuget_dependency_solver | true/false | | ## Unsupported features and configurations -We aim to support all official features and configuration options, but there are some current limitations and exceptions. +We aim to support all [official configuration options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file), but there are some limitations for: ### Extension Task #### `dependabot@V2` -- [`schedule` config options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#scheduleinterval) are ignored, use [pipeline scheduled triggers](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/scheduled-triggers?view=azure-devops&tabs=yaml#scheduled-triggers) instead. -- [Security updates only](https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates#overriding-the-default-behavior-with-a-configuration-file) (i.e. `open-pull-requests-limit: 0`) are not supported. _(coming soon)_ +- [`schedule`](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#scheduleinterval) is ignored, use [pipeline scheduled triggers](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/scheduled-triggers?view=azure-devops&tabs=yaml#scheduled-triggers) instead. +- [Security-only updates](https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates#overriding-the-default-behavior-with-a-configuration-file) (`open-pull-requests-limit: 0`) are not supported. _(coming soon)_ #### `dependabot@V1` -- [`schedule` config options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#scheduleinterval) are ignored, use [pipeline scheduled triggers](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/scheduled-triggers?view=azure-devops&tabs=yaml#scheduled-triggers) instead. -- [`directories` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#directories) is only supported if task input `useUpdateScriptVNext: true` is set. -- [`groups` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#groups) is only supported if task input `useUpdateScriptVNext: true` is set. -- [`ignore` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#ignore) may not behave to official specifications unless task input `useUpdateScriptVNext: true` is set. If you are having issues, search for related issues such as before creating a new issue. -- Private feed/registry authentication is known to cause errors with some package ecyosystems. Support is _slightly_ improved when task input `useUpdateScriptVNext: true` is set, but not still not fully supported. See [problems with authentication](./docs/migrations/v1-to-v2.md#resolving-private-feedregistry-authentication-issues) for more. +- [`schedule`](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#scheduleinterval) is ignored, use [pipeline scheduled triggers](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/scheduled-triggers?view=azure-devops&tabs=yaml#scheduled-triggers) instead. +- [`directories`](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#directories) are only supported if task input `useUpdateScriptVNext: true` is set. +- [`groups`](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#groups) are only supported if task input `useUpdateScriptVNext: true` is set. +- [`ignore`](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#ignore) may not behave to official specifications unless task input `useUpdateScriptVNext: true` is set. If you are having issues, search for related issues such as before creating a new issue. +- Private feed/registry authentication may not work with all package ecyosystems. Support is _slightly_ improved when task input `useUpdateScriptVNext: true` is set, but not still not fully supported. See [problems with authentication](https://github.com/tinglesoftware/dependabot-azure-devops/discussions/1317) for more. ### Updater Docker image -- Private feed/registry authentication is known to cause errors with some package ecyosystems. See [problems with authentication](./docs/migrations/v1-to-v2.md#resolving-private-feedregistry-authentication-issues) for more. +- Private feed/registry authentication may not work with all package ecyosystems. See [problems with authentication](https://github.com/tinglesoftware/dependabot-azure-devops/discussions/1317) for more. ### Server -- [`directories` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#directories) is not supported. -- [`groups` config option](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#groups) is not supported. -- Private feed/registry authentication is known to cause errors with some package ecyosystems. See [problems with authentication](./docs/migrations/v1-to-v2.md#resolving-private-feedregistry-authentication-issues) for more. +- [`directories`](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#directories) are not supported. +- [`groups`](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#groups) are not supported. +- Private feed/registry authentication may not work with all package ecyosystems. See [problems with authentication](https://github.com/tinglesoftware/dependabot-azure-devops/discussions/1317) for more. ## Migration Guide - [Extension Task V1 → V2](./docs/migrations/v1-to-v2) diff --git a/docs/migrations/v1-to-v2.md b/docs/migrations/v1-to-v2.md index c77feb67..330f36f4 100644 --- a/docs/migrations/v1-to-v2.md +++ b/docs/migrations/v1-to-v2.md @@ -1,20 +1,65 @@ +> [!WARNING] +> **This is a work in progress;** `dependabot@V2` is still under development and this document may change without notice up until general availability (GA). + # Table of Contents - [Summary of changes V1 → V2](#summary-of-changes-v1-v2) * [Resolving private feed/registry authentication issues](#resolving-private-feedregistry-authentication-issues) - [Breaking changes V1 → V2](#breaking-changes-v1-v2) - [Steps to migrate V1 → V2](#steps-to-migrate-v1-v2) +- [Todo before general availability](#todo-before-general-availability) # Summary of changes V1 → V2 -... +V2 is a complete re-write of the Dependabot task; It aims to: + +- Resolve the [numerous private feed/registry authentication issues](https://github.com/tinglesoftware/dependabot-azure-devops/discussions/1317) that currently exist in V1; +- More closely align the update logic with the GitHub-hosted Dependabot service; -See [extension task architecture](../extension.md#architecture) for more technical details. +The task now uses [Dependabot CLI](https://github.com/dependabot/cli) to perform dependency updates, which is the _[current]_ recommended approach for running Dependabot. -## Resolving private feed/registry authentication issues -... +See [extension task architecture](../extension.md#architecture) for more technical details on changes to the update process. # Breaking changes V1 → V2 -... -# Steps to migrate V1 → V2 -... +### New pipeline agent requirements; "Go" must be installed +Dependabot CLI requires [Go](https://go.dev/doc/install) (1.22+) and [Docker](https://docs.docker.com/get-started/get-docker/) (with Linux containers). +If you use [Microsoft-hosted agents](https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops&tabs=yaml#software), we recommend using the [ubuntu-latest](https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md) image, which meets all task requirements. +For self-hosted agents, you will need to install Go 1.22+. + +### Security-only updates and "fixed vulnerabilities" are not implemented (yet) +Using configuration `open-pull-requests-limit: 0` will cause a "not implemented" error. This is [current limitation of V2](../../README.md#unsupported-features-and-configurations). A solution is still under development and is expected to be resolved before general availability. +See: https://github.com/dependabot/cli/issues/360 for more technical details. + +### Task Input `updaterOptions` has been renamed to `experiments` +Renamed to match Dependabot Core/CLI terminology. The input value remains unchanged. See [configuring experiments](../../README.md#configuring-experiments) for more details. + +### Task Input `failOnException` has been removed +Due to the design of Dependabot CLI, the update process can no longer be interrupted once the update has started. Because of this, the update will now continue on error and summarise all error at the end of the update process. + +### Task Input `excludeRequirementsToUnlock` has been removed +This was a customisation/workaround specific to the V1 update script that can no longer be implemented with Dependabot CLI as it is not an official configuration option. + +### Task Input `dockerImageTag` has been removed +This is no longer required as the [custom] [Dependabot Updater image](../updater.md) is no longer used. + +### Task Input `extraEnvironmentVariables` has been removed +Due to the containerised design of Dependabot CLI, environment variables can no longer be passed from the task to the updater process. All Dependabot config must now set via `dependabot.yaml` or as task inputs. The following old environment variables have been converted to task inputs: + +| Environment Variable | New Task Input | +|--|--| +|DEPENDABOT_AUTHOR_EMAIL|authorEmail| +|DEPENDABOT_AUTHOR_NAME|authorName| + + +## Todo before general availability +Before removing the preview flag from V2 `task.json`, we need to: + - [x] Open an issue in Dependabot-CLI, enquire how security-advisories are expected to be provided **before** knowing the list of dependencies. (https://github.com/dependabot/cli/issues/360) + - [ ] Convert GitHub security advisory client in `vulnerabilities.rb` to TypeScript code + - [ ] Implement `security-advisories` config once the answer the above is known + - [x] Review `task.json`, add documentation for new V2 inputs + - [x] Update `\docs\extension.md` with V2 docs + - [x] Update `\extension\README.MD` with V2 docs + - [x] Update `\README.MD` with V2 docs + - [ ] Do a general code tidy-up pass (check all "TODO" comments) + - [ ] Add unit tests for V2 utils scripts + - [ ] Investigate https://zod.dev/ \ No newline at end of file diff --git a/docs/updater.md b/docs/updater.md index 2e66f76f..117cae62 100644 --- a/docs/updater.md +++ b/docs/updater.md @@ -1,6 +1,6 @@ > [!WARNING] -> Use of the Dependabot Updater image is no longer recommended since v2.0; This updater image is considered an internal component within Dependabot and is not intended to be run directly without the use of a credentials proxy. See [unsupported features and configuration](../README.md#unsupported-features-and-configurations) for more details on the limitations of this image. +> **Deprecated;** Use of the Dependabot Updater image is no longer recommended since v2.0; The "updater" component is considered an internal to Dependabot and is not intended to be run directly by end-users. There are known limitations with this image, see [unsupported features and configuration](../README.md#unsupported-features-and-configurations) for more details. # Table of Contents @@ -140,7 +140,7 @@ The following environment variables are required when running the container. Install [Docker](https://docs.docker.com/engine/install/) and [Ruby](https://www.ruby-lang.org/en/documentation/installation/). -> [!TIP] +> [!NOTE] > If developing in Linux, you'll also need the the build essentials and Ruby development packages; These are typically `build-essentials` and `ruby-dev`. Install the project build tools using Bundle: diff --git a/extension/README.md b/extension/README.md index 66f4680a..dea9b471 100644 --- a/extension/README.md +++ b/extension/README.md @@ -56,8 +56,8 @@ Dependabot uses Docker containers, which may take time to install if not already |autoCompleteIgnoreConfigIds|**_Optional_**. List of any policy configuration Id's which auto-complete should not wait for. Only applies to optional policies. Auto-complete always waits for required (blocking) policies.| |autoApprove|**_Optional_**. Determines if the pull requests that dependabot creates should be automatically completed. When set to `true`, pull requests will be approved automatically. To use a different user for approval, supply `autoApproveUserToken` input. Defaults to `false`.| |autoApproveUserToken|**_Optional_**. A personal access token for the user to automatically approve the created PR.| -|authorEmail|**_Optional_**. The email address to use for the change commit author. Can be used to associate the committer with an existing account, to provide a profile picture.| -|authorName|**_Optional_**. The display name to use for the change commit author.| +|authorEmail|**_Optional_**. The email address to use for the change commit author. Can be used to associate the committer with an existing account, to provide a profile picture. Defaults to `noreply@github.com`.| +|authorName|**_Optional_**. The name to use as the git commit author of the pull requests. Defaults to `dependabot[bot]`.| |securityAdvisoriesFile|**_Optional_**. The path to a JSON file containing additional security advisories to be included when performing package updates. See: [Configuring security advisories and known vulnerabilities](https://github.com/tinglesoftware/dependabot-azure-devops/#configuring-security-advisories-and-known-vulnerabilities).| |azureDevOpsServiceConnection|**_Optional_**. A Service Connection to use for accessing Azure DevOps. Supply a value here to avoid using permissions for the Build Service either because you cannot change its permissions or because you prefer that the Pull Requests be done by a different user. When not provided, the current authentication scope is used.
See the [documentation](https://learn.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops) to know more about creating a Service Connections| |azureDevOpsAccessToken|**_Optional_**. The Personal Access Token for accessing Azure DevOps. Supply a value here to avoid using permissions for the Build Service either because you cannot change its permissions or because you prefer that the Pull Requests be done by a different user. When not provided, the current authentication scope is used. In either case, be use the following permissions are granted:
- Code (Full)
- Pull Requests Threads (Read & Write).
See the [documentation](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#create-a-pat) to know more about creating a Personal Access Token.
Use this in place of `azureDevOpsServiceConnection` such as when it is not possible to create a service connection.| diff --git a/extension/tasks/dependabot/dependabotV2/task.json b/extension/tasks/dependabot/dependabotV2/task.json index d939576b..978e213a 100644 --- a/extension/tasks/dependabot/dependabotV2/task.json +++ b/extension/tasks/dependabot/dependabotV2/task.json @@ -138,7 +138,7 @@ "label": "Git commit uthor email address", "defaultValue": "", "required": false, - "helpMarkDown": "The email address to use as the git commit author of the pull requests. Defaults to 'noreply@github.com'." + "helpMarkDown": "The email address to use for the change commit author. Can be used to associate the committer with an existing account, to provide a profile picture. Defaults to `noreply@github.com`." }, { "name": "authorName", @@ -147,7 +147,7 @@ "label": "Git commit author name", "defaultValue": "", "required": false, - "helpMarkDown": "The name to use as the git commit author of the pull requests. Defaults to 'dependabot[bot]'." + "helpMarkDown": "The name to use as the git commit author of the pull requests. Defaults to `dependabot[bot]`." }, { @@ -202,7 +202,7 @@ "label": "Monitor the discovered dependencies", "defaultValue": false, "required": false, - "helpMarkDown": "When set to `true`, the discovered dependencies will be stored against the project. Defaults to `false`." + "helpMarkDown": "Determines if the last know dependency list information should be stored in the parent DevOps project properties. If enabled, the authenticated user must have the `Project & Team (Write)` permission for the project. Enabling this option improves performance when doing security-only updates. Defaults to `false`." }, { "name": "targetRepositoryName", @@ -227,7 +227,7 @@ "groupName": "advanced", "label": "Dependabot updater experiments", "required": false, - "helpMarkDown": "Comma-seperated list of Dependabot experiments. Available options depend on the ecosystem." + "helpMarkDown": "Comma-seperated list of key/value pairs representing the enabled Dependabot experiments e.g. `experiments: 'tidy=true,vendor=true,goprivate=*'`. Available options vary depending on the package ecosystem. See [configuring experiments](https://github.com/tinglesoftware/dependabot-azure-devops/#configuring-experiments) for more details." } ], "dataSourceBindings": [], From ecca888a691ac09c629b9b69be7537c25ff75d5c Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Fri, 20 Sep 2024 12:19:56 +1200 Subject: [PATCH 39/57] Typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 74403c55..94dfd6c6 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Unlike the GitHub-hosted version, Dependabot for Azure DevOps must be explicitly Similar to the GitHub-hosted version, Dependabot is configured using a [dependabot.yml file](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file) located at `.azuredevops/dependabot.yml` or `.github/dependabot.yml` in your repository. -Most [official configuration options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file) are supported since V2; Earlier versions have serveral limitations, see [unsupported features and configurations](#unsupported-features-and-configurations) for more. +Most [official configuration options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file) are supported since V2; Earlier versions have several limitations, see [unsupported features and configurations](#unsupported-features-and-configurations) for more. ## Configuring private feeds and registries From 4f693e46e209fba5cb6771115018ab5ccc53d0c8 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Fri, 20 Sep 2024 16:44:52 +1200 Subject: [PATCH 40/57] Update documentation --- README.md | 30 +++++++++++++++++++----------- docs/extension.md | 3 ++- docs/migrations/v1-to-v2.md | 8 ++------ docs/updater.md | 10 ++++++++-- extension/README.md | 12 +++++++++--- 5 files changed, 40 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 94dfd6c6..85baf0ba 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,9 @@ In this repository you'll find: 1. Dependabot Server, [source code](./server/) and [docs](./docs/server.md). 1. Dependabot Updater image, [Dockerfile](./updater/Dockerfile), [source code](./updater/) and [docs](./docs/updater.md). **(Deprecated since v2.0)** +> [!IMPORTANT] +> This project is currently undergoing a major version increment (V1 → V2); See the [migration guide](./docs/migrations/v1-to-v2.md#summary-of-changes-v1--v2) for more details and progress updates. + ## Table of Contents - [Getting started](#getting-started) - [Using a configuration file](#using-a-configuration-file) @@ -31,14 +34,14 @@ In this repository you'll find: ## Getting started -Unlike the GitHub-hosted version, Dependabot for Azure DevOps must be explicitly setup in your organisation, creating a `dependabot.yml` file alone will **not** enable updates. There are two ways to enable Dependabot: +Unlike the GitHub-hosted version, Dependabot for Azure DevOps must be explicitly setup in your organisation; creating a `dependabot.yml` file alone is **not** enough to enable updates. There are two ways to enable Dependabot, using: -- [Azure DevOps Extension](https://marketplace.visualstudio.com/items?itemName=tingle-software.dependabot) - Ideal if you want to get Dependabot running with minimal administrative effort. The extension can run directly inside your existing pipeline agents and doesn't require hosting of any additional services. Because the extension runs in pipelines, this option does not scale well if you have a large number of projects/repositories. +- [Azure DevOps Extension](https://marketplace.visualstudio.com/items?itemName=tingle-software.dependabot) - Ideal if you want to get Dependabot running with minimal administrative effort. The extension can run directly inside your existing pipeline agents and doesn't require hosting of any additional services. Because the extension runs in pipelines, this option does **not** scale well if you have a large number of projects and repositories. -- [Hosted Server](./docs/server.md) - Ideal if you have a large number of projects/repositories or prefer to run Dependabot as a managed service instead of using pipeline agents. See [why should I use the server?](./docs/server.md#why-should-i-use-the-server) +- [Hosted Server](./docs/server.md) - Ideal if you have a large number of projects and repositories or prefer to run Dependabot as a managed service instead of using pipeline agents. See [why should I use the server?](./docs/server.md#why-should-i-use-the-server) for more info. > [!NOTE] -> A hosted version is available to sponsors (most, but not all). It includes hassle free runs where the infrastructure is maintained for you. Much like the GitHub hosted version. Alternatively, you can run and host your own [self-hosted server](./docs/server.md). Once you sponsor, you can send out an email to an maintainer or wait till they reach out. This is meant to ease the burden until GitHub/Azure/Microsoft can get it working natively (which could also be never) and hopefully for free. +> A hosted version is available to sponsors (most, but not all). It includes hassle free runs where the infrastructure is maintained for you. Much like the GitHub hosted version. Alternatively, you can run and host your own [self-hosted server](./docs/server.md). Once you sponsor, you can send out an email to a maintainer or wait till they reach out. This is meant to ease the burden until GitHub/Azure/Microsoft can get it working natively (which could also be never) and hopefully for free. ## Using a configuration file @@ -136,20 +139,20 @@ BUT the values will be used from pipeline environment variables. Template variab ## Configuring security advisories and known vulnerabilities -Security-only updates is a mechanism to only create pull requests for dependencies with vulnerabilities by updating them to the earliest available non-vulnerable version. Security updates are supported in the same way as the GitHub-hosted version provided that a GitHub access token with `public_repo` access is provided in the `gitHubConnection` task input. +Security-only updates is a mechanism to only create pull requests for dependencies with vulnerabilities by updating them to the earliest available non-vulnerable version. [Security updates are supported in the same way as the GitHub-hosted version](https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates#overriding-the-default-behavior-with-a-configuration-file) provided that a GitHub access token with `public_repo` access is provided in the `gitHubAccessToken` or `gitHubConnection` task inputs. -You can provide extra security advisories, such as those for an internal dependency, in a JSON file via the `securityAdvisoriesFile` task input e.g. `securityAdvisoriesFile: '$(Pipeline.Workspace)/advisories.json'`. An example file is available [here](./advisories-example.json). +You can provide extra security advisories, such as those for an internal dependency, in a JSON file via the `securityAdvisoriesFile` task input e.g. `securityAdvisoriesFile: '$(Pipeline.Workspace)/advisories.json'`. An example file is available in [./advisories-example.json](./advisories-example.json). ## Configuring experiments -Dependabot uses an internal feature flag system called "experiments". Typically, experiments represent new features or changes in logic which are still being ]internal] tested before becoming generally available. In some cases, you may want to opt-in to experiments to work around known issues or to opt-in to preview features. +Dependabot uses an internal feature flag system called "experiments". Typically, experiments represent new features or changes in logic which are still being internally tested before becoming generally available. In some cases, you may want to opt-in to experiments to work around known issues or to opt-in to preview features ahead of general availability (GA). Experiments vary depending on the package ecyosystem used; They can be enabled using the `experiments` task input with a comma-seperated list of key/value pairs representing the experiments e.g. `experiments: 'tidy=true,vendor=true,goprivate=*'`. -> [!WARNING] -> For convenience, known experiments as of v0.275.0 are listed below; **This be out-of-date at the time of reading.** +> [!NOTE] +> Dependabot experinment names are not [publicly] documented. For convenience, some known experiments are listed below; However, **be aware that this may be out-of-date at the time of reading.** -> [!TIP] -> To find the latest list of Dependabot experiments, search the `dependabot-core` GitHub repository using queries like ["enabled?(x)"](https://github.com/search?q=repo%3Adependabot%2Fdependabot-core+%2Fenabled%5CW%5C%28.*%5C%29%2F&type=code) and ["options.fetch(x)"](https://github.com/search?q=repo%3Adependabot%2Fdependabot-core+%2Foptions%5C.fetch%5C%28.*%2C%2F&type=code). +
+List of known experiments from dependabot-core@0.275.0 |Package Ecosystem|Experiment Name|Value Type|Description| |--|--|--|--| @@ -169,6 +172,11 @@ Experiments vary depending on the package ecyosystem used; They can be enabled u | NuGet | nuget_native_analysis | true/false | | | NuGet | nuget_dependency_solver | true/false | | +
+ +> [!TIP] +> To find the latest list of Dependabot experiments, search the `dependabot-core` GitHub repository using queries like ["enabled?(x)"](https://github.com/search?q=repo%3Adependabot%2Fdependabot-core+%2Fenabled%5CW%5C%28.*%5C%29%2F&type=code) and ["options.fetch(x)"](https://github.com/search?q=repo%3Adependabot%2Fdependabot-core+%2Foptions%5C.fetch%5C%28.*%2C%2F&type=code). + ## Unsupported features and configurations We aim to support all [official configuration options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file), but there are some limitations for: diff --git a/docs/extension.md b/docs/extension.md index 247f6f02..e90fafd4 100644 --- a/docs/extension.md +++ b/docs/extension.md @@ -68,7 +68,8 @@ npm test # Architecture ## Task V2 high-level update process diagram -High-level sequence diagram illustrating how the `dependabotV2` task performs updates using [dependabot-cli](https://github.com/dependabot/cli). See [how dependabot-cli works](https://github.com/dependabot/cli?tab=readme-ov-file#how-it-works) for more details. +High-level sequence diagram illustrating how the `dependabotV2` task performs updates using [dependabot-cli](https://github.com/dependabot/cli). For more technical details, see [how dependabot-cli works](https://github.com/dependabot/cli?tab=readme-ov-file#how-it-works). + ```mermaid sequenceDiagram participant ext as Dependabot DevOps Extension diff --git a/docs/migrations/v1-to-v2.md b/docs/migrations/v1-to-v2.md index 330f36f4..962b5531 100644 --- a/docs/migrations/v1-to-v2.md +++ b/docs/migrations/v1-to-v2.md @@ -1,12 +1,10 @@ > [!WARNING] -> **This is a work in progress;** `dependabot@V2` is still under development and this document may change without notice up until general availability (GA). +> **:construction: Work in progress;** `dependabot@V2` is still under development and this document may change without notice up until general availability (GA). # Table of Contents - [Summary of changes V1 → V2](#summary-of-changes-v1-v2) - * [Resolving private feed/registry authentication issues](#resolving-private-feedregistry-authentication-issues) - [Breaking changes V1 → V2](#breaking-changes-v1-v2) -- [Steps to migrate V1 → V2](#steps-to-migrate-v1-v2) - [Todo before general availability](#todo-before-general-availability) # Summary of changes V1 → V2 @@ -15,9 +13,7 @@ V2 is a complete re-write of the Dependabot task; It aims to: - Resolve the [numerous private feed/registry authentication issues](https://github.com/tinglesoftware/dependabot-azure-devops/discussions/1317) that currently exist in V1; - More closely align the update logic with the GitHub-hosted Dependabot service; -The task now uses [Dependabot CLI](https://github.com/dependabot/cli) to perform dependency updates, which is the _[current]_ recommended approach for running Dependabot. - -See [extension task architecture](../extension.md#architecture) for more technical details on changes to the update process. +The task now uses [Dependabot CLI](https://github.com/dependabot/cli) to perform dependency updates, which is the _[currently]_ recommended approach for running Dependabot. See [extension task architecture](../extension.md#architecture) for more details on the technical changes and impact to the update process. # Breaking changes V1 → V2 diff --git a/docs/updater.md b/docs/updater.md index 117cae62..26ab4ba6 100644 --- a/docs/updater.md +++ b/docs/updater.md @@ -41,7 +41,8 @@ docker run --rm -t \ ghcr.io/tinglesoftware/dependabot-updater- update_script ``` -An example, for Azure DevOps Services: +
+Example, for Azure DevOps Services ```bash docker run --rm -t \ @@ -58,7 +59,10 @@ docker run --rm -t \ ghcr.io/tinglesoftware/dependabot-updater-nuget update_script ``` -An example, for Azure DevOps Server: +
+ +
+Example, for Azure DevOps Server ```bash docker run --rm -t \ @@ -78,6 +82,8 @@ docker run --rm -t \ ghcr.io/tinglesoftware/dependabot-updater-nuget update_script ``` +
+ ## Environment Variables The following environment variables are required when running the container. diff --git a/extension/README.md b/extension/README.md index dea9b471..236d2d63 100644 --- a/extension/README.md +++ b/extension/README.md @@ -37,14 +37,15 @@ steps: ## Task Requirements -The task makes use of [dependabot-cli](https://github.com/dependabot/cli), which requires [Go](https://go.dev/doc/install) (1.22+) and [Docker](https://docs.docker.com/get-started/get-docker/) (with Linux containers) be installed on the pipeline agent. +The task uses [dependabot-cli](https://github.com/dependabot/cli), which requires [Go](https://go.dev/doc/install) (1.22+) and [Docker](https://docs.docker.com/get-started/get-docker/) (with Linux containers) be installed on the pipeline agent. If you use [Microsoft-hosted agents](https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops&tabs=yaml#software), we recommend using the [ubuntu-latest](https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md) image, which meets all task requirements. Dependabot uses Docker containers, which may take time to install if not already cached. Subsequent dependabot tasks in the same job will be faster after initially pulling the images. An alternative way to run your pipelines faster is by leveraging Docker caching in Azure Pipelines (See [#113](https://github.com/tinglesoftware/dependabot-azure-devops/issues/113#issuecomment-894771611)). ## Task Parameters -### `dependabot@V2` +
+dependabot@V2 |Input|Description| |--|--| @@ -68,7 +69,10 @@ Dependabot uses Docker containers, which may take time to install if not already |targetUpdateIds|**_Optional_**. A semicolon (`;`) delimited list of update identifiers run. Index are zero-based and in the order written in the configuration file. When not present, all the updates are run. This is meant to be used in scenarios where you want to run updates a different times from the same configuration file given you cannot schedule them independently in the pipeline.| |experiments|**_Optional_**. Comma separated list of Dependabot experiments; available options depend on the ecosystem. Example: `tidy=true,vendor=true,goprivate=*`. See: [Configuring experiments](https://github.com/tinglesoftware/dependabot-azure-devops/#configuring-experiments)| -### `dependabot@V1` **(Deprecated)** +
+ +
+dependabot@V1 (Deprecated) |Input|Description| |--|--| @@ -94,6 +98,8 @@ Dependabot uses Docker containers, which may take time to install if not already |dockerImageTag|**_Optional_**. The image tag to use when pulling the docker container used by the task. A tag also defines the version. By default, the task decides which tag/version to use. This can be the latest or most stable version. When not provided, the value is inferred from the current task version| |extraEnvironmentVariables|**_Optional_**. A semicolon (`;`) delimited list of environment variables that are sent to the docker container. See possible use case [here](https://github.com/tinglesoftware/dependabot-azure-devops/issues/138)| +
+ ## Advanced - [Configuring private feeds and registries](https://github.com/tinglesoftware/dependabot-azure-devops/#configuring-private-feeds-and-registries) From 0be8391c2e90eddc8da6d05b6da3f70fff8dc2a5 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sat, 21 Sep 2024 19:02:42 +1200 Subject: [PATCH 41/57] Fix reference to undefined `this.cachedUserIds` --- .../dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts index 2118d45d..fd0fbf70 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts @@ -23,6 +23,7 @@ export class AzureDevOpsWebApiClient { organisationApiUrl, getPersonalAccessTokenHandler(accessToken) ); + this.cachedUserIds = {}; } /** @@ -34,7 +35,7 @@ export class AzureDevOpsWebApiClient { // If no email is provided, resolve to the authenticated user if (!email) { - this.cachedUserIds[this.accessToken] ||= (await this.connection.connect()).authenticatedUser?.id || ""; + this.cachedUserIds[this.accessToken] ||= ((await this.connection.connect())?.authenticatedUser?.id || ""); return this.cachedUserIds[this.accessToken]; } From 8ea41adb8496dd3e129fd83044e0ec65f48e4eb7 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sat, 21 Sep 2024 19:53:47 +1200 Subject: [PATCH 42/57] Use case insensitive comparision when parsing "System.Debug" variable --- .../tasks/dependabot/dependabotV1/utils/getSharedVariables.ts | 2 +- .../tasks/dependabot/dependabotV2/utils/getSharedVariables.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV1/utils/getSharedVariables.ts b/extension/tasks/dependabot/dependabotV1/utils/getSharedVariables.ts index 87a90ab6..56383d79 100644 --- a/extension/tasks/dependabot/dependabotV1/utils/getSharedVariables.ts +++ b/extension/tasks/dependabot/dependabotV1/utils/getSharedVariables.ts @@ -122,7 +122,7 @@ export default function getSharedVariables(): ISharedVariables { let excludeRequirementsToUnlock = tl.getInput('excludeRequirementsToUnlock') || ''; let updaterOptions = tl.getInput('updaterOptions'); - let debug: boolean = tl.getVariable('System.Debug')?.localeCompare('true') === 0; + let debug: boolean = tl.getVariable('System.Debug')?.match(/true/i) ? true : false; // Get the target identifiers let targetUpdateIds = tl.getDelimitedInput('targetUpdateIds', ';', false).map(Number); diff --git a/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts b/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts index af268204..ee5010b1 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/getSharedVariables.ts @@ -127,7 +127,7 @@ export default function getSharedVariables(): ISharedVariables { {} as Record ); - let debug: boolean = tl.getVariable('System.Debug')?.localeCompare('true') === 0; + let debug: boolean = tl.getVariable('System.Debug')?.match(/true/i) ? true : false; // Get the target identifiers let targetUpdateIds = tl.getDelimitedInput('targetUpdateIds', ';', false).map(Number); From c031b471388b08ce0c197537c8264277603cfca7 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sat, 21 Sep 2024 20:08:16 +1200 Subject: [PATCH 43/57] Fix dependabot tool path detection in agents where `$PATH` does not contain `$GOPATH/bin` --- .../azure-devops/AzureDevOpsWebApiClient.ts | 2 +- .../utils/dependabot-cli/DependabotCli.ts | 30 +++++++++++-------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts index fd0fbf70..9a862a1a 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts @@ -79,7 +79,7 @@ export class AzureDevOpsWebApiClient { * @returns */ public async getActivePullRequestProperties(project: string, repository: string, creator: string): Promise { - console.info(`Fetching active pull request properties in '${project}/${repository}' for '${creator}'...`); + console.info(`Fetching active pull request properties in '${project}/${repository}' for user id '${creator}'...`); try { const git = await this.connection.getGitApi(); const pullRequests = await git.getPullRequests( diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts index 48a1d490..4f213c15 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts @@ -44,8 +44,8 @@ export class DependabotCli { } ): Promise { - // Install dependabot if not already installed - await this.ensureToolsAreInstalled(); + // Find the dependabot tool path, or install it if missing + const dependabotPath = await this.getDependabotToolPath(); // Create the job directory const jobId = operation.job.id; @@ -79,9 +79,8 @@ export class DependabotCli { // Run dependabot update if (!fs.existsSync(jobOutputPath) || fs.statSync(jobOutputPath)?.size == 0) { console.info(`Running Dependabot update job from '${jobInputPath}'...`); - const dependabotTool = tool(which("dependabot", true)).arg(dependabotArguments); + const dependabotTool = tool(dependabotPath).arg(dependabotArguments); const dependabotResultCode = await dependabotTool.execAsync({ - silent: !this.debug, failOnStdErr: false, ignoreReturnCode: true }); @@ -126,20 +125,27 @@ export class DependabotCli { return operationResults.length > 0 ? operationResults : undefined; } - // Install dependabot if not already installed - private async ensureToolsAreInstalled(): Promise { + // Get the dependabot tool path and install if missing + private async getDependabotToolPath(installIfMissing: boolean = true): Promise { debug('Checking for `dependabot` install...'); - if (which("dependabot", false)) { - return; + let dependabotPath = which("dependabot", false); + if (dependabotPath) { + return dependabotPath; + } + if (!installIfMissing) { + throw new Error("Dependabot CLI install not found"); } - console.info("Dependabot CLI install was not found, installing now with `go install`..."); + console.info("Dependabot CLI install was not found, installing now with `go install dependabot`..."); const goTool: ToolRunner = tool(which("go", true)); goTool.arg(["install", this.toolImage]); - goTool.execSync({ - silent: !this.debug - }); + goTool.execSync(); + + // Depending on how go is installed on the host agent, the go bin path may not be in the PATH environment variable. + // If `which("dependabot")` still doesn't resolve, we must manually resolve the path; It will either be "$GOPATH/bin/dependabot" or "$HOME/go/bin/dependabot" if $GOPATH is not set. + const goBinPath = process.env.GOPATH ? path.join(process.env.GOPATH, 'bin') : path.join(os.homedir(), 'go', 'bin'); + return which("dependabot", false) || path.join(goBinPath, 'dependabot'); } // Create the jobs directory if it does not exist From c69a3f6e864f35c133d8e2f534049256dbc59a65 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sat, 21 Sep 2024 20:23:28 +1200 Subject: [PATCH 44/57] Add more logging --- .../utils/dependabot-cli/DependabotOutputProcessor.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index 5560c602..38e1f84f 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -74,6 +74,7 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess return JSON.stringify(repoDependencyLists); } ); + console.info(`Dependency list snapshot was updated for project '${project}'`); } return true; From d361188820b73437bef84acebdfed00aef21c455 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sat, 21 Sep 2024 20:24:08 +1200 Subject: [PATCH 45/57] Fix for task reporting success when pull request creation failed --- .../utils/dependabot-cli/DependabotOutputProcessor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index 38e1f84f..48df28fa 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -133,7 +133,7 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess }); } - return newPullRequestId !== undefined; + return newPullRequestId > 0; case 'update_pull_request': if (this.taskInputs.skipPullRequests) { From 440fda3c3e6ecb090537c7f5399857ec8d5044ba Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sat, 21 Sep 2024 22:50:33 +1200 Subject: [PATCH 46/57] Add more logging; Fix formatting --- .../azure-devops/AzureDevOpsWebApiClient.ts | 12 +++++++----- .../dependabot-cli/DependabotJobBuilder.ts | 18 +++++++++--------- .../DependabotOutputProcessor.ts | 13 ++++++------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts index 9a862a1a..1d8b06cc 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts @@ -251,7 +251,7 @@ export class AzureDevOpsWebApiClient { ); } - console.info(` - Pull request ${pullRequest.pullRequestId} was created successfully.`); + console.info(` - Pull request #${pullRequest.pullRequestId} was created successfully.`); return pullRequest.pullRequestId; } catch (e) { @@ -356,7 +356,7 @@ export class AzureDevOpsWebApiClient { const git = await this.connection.getGitApi(); // Approve the pull request - console.info(` - Approving pull request...`); + console.info(` - Creating reviewer vote on pull request...`); await git.createPullRequestReviewer( { vote: 10, // 10 - approved 5 - approved with suggestions 0 - no vote -5 - waiting for author -10 - rejected @@ -367,6 +367,8 @@ export class AzureDevOpsWebApiClient { userId, options.project ); + + console.info(` - Pull request #${options.pullRequestId} was approved.`); } catch (e) { error(`Failed to approve pull request: ${e}`); @@ -451,7 +453,7 @@ export class AzureDevOpsWebApiClient { return false; } } - + /** * Get project properties * @param project @@ -482,14 +484,14 @@ export class AzureDevOpsWebApiClient { */ public async updateProjectProperty(project: string, name: string, valueBuilder: (existingValue: string) => string): Promise { try { - + // Get the existing project property value const core = await this.connection.getCoreApi(); const projects = await core.getProjects(); const projectGuid = projects?.find(p => p.name === project)?.id; const properties = await core.getProjectProperties(projectGuid); const propertyValue = properties?.find(p => p.name === name)?.value; - + // Update the project property await core.setProjectProperties( undefined, diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts index 779730cb..d6e9fb3b 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts @@ -27,15 +27,15 @@ export class DependabotJobBuilder { ): IDependabotUpdateOperation { const packageEcosystem = update["package-ecosystem"]; const securityUpdatesOnly = update["open-pull-requests-limit"] == 0; - const updateDependencyNames = securityUpdatesOnly ? mapDependenciesForSecurityUpdate(dependencyList): undefined; + const updateDependencyNames = securityUpdatesOnly ? mapDependenciesForSecurityUpdate(dependencyList) : undefined; return buildUpdateJobConfig( `update-${packageEcosystem}-${securityUpdatesOnly ? 'security-only' : 'all'}`, - taskInputs, - update, - registries, - false, - undefined, - updateDependencyNames, + taskInputs, + update, + registries, + false, + undefined, + updateDependencyNames, existingPullRequests ); } @@ -141,7 +141,7 @@ function mapDependenciesForSecurityUpdate(dependencyList: any[]): string[] { ); // Attempt to do a security update for "all dependencies"; it will probably fail this is not supported in dependabot-updater yet, but it is best we can do... - return []; + return []; } // Return only dependencies that are vulnerable, ignore the rest @@ -203,7 +203,7 @@ function mapVersionStrategyToRequirementsUpdateStrategy(versioningStrategy: stri if (!versioningStrategy) { return undefined; } - switch(versioningStrategy) { + switch (versioningStrategy) { case 'auto': return undefined; case 'increase': return 'bump_versions'; case 'increase-if-necessary': return 'bump_versions_if_necessary'; diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index 48df28fa..ac5bf9e2 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -20,7 +20,7 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess // Custom properties used to store dependabot metadata in projects. // https://learn.microsoft.com/en-us/rest/api/azure/devops/core/projects/set-project-properties public static PROJECT_PROPERTY_NAME_DEPENDENCY_LIST = "Dependabot.DependencyList"; - + // Custom properties used to store dependabot metadata in pull requests. // https://learn.microsoft.com/en-us/rest/api/azure/devops/git/pull-request-properties public static PR_PROPERTY_NAME_PACKAGE_MANAGER = "Dependabot.PackageManager"; @@ -56,13 +56,12 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess case 'update_dependency_list': // Store the dependency list snapshot in project properties, if configured - if (this.taskInputs.storeDependencyList) - { + if (this.taskInputs.storeDependencyList) { console.info(`Storing the dependency list snapshot for project '${project}'...`); await this.prAuthorClient.updateProjectProperty( project, DependabotOutputProcessor.PROJECT_PROPERTY_NAME_DEPENDENCY_LIST, - function(existingValue: string) { + function (existingValue: string) { const repoDependencyLists = JSON.parse(existingValue || '{}'); repoDependencyLists[repository] = repoDependencyLists[repository] || {}; repoDependencyLists[repository][update.job["package-manager"]] = { @@ -70,13 +69,13 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess 'dependency-files': data['dependency_files'], 'last-updated': new Date().toISOString() }; - + return JSON.stringify(repoDependencyLists); } ); console.info(`Dependency list snapshot was updated for project '${project}'`); } - + return true; case 'create_pull_request': @@ -184,7 +183,7 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess // TODO: GitHub Dependabot will close with reason "Superseded by ${new_pull_request_id}" when another PR supersedes it. // How do we detect this? Do we need to? - + // Close the pull request return await this.prAuthorClient.closePullRequest({ project: project, From 8d557606601d1b5ac016060314bfb05d662e0587 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sat, 21 Sep 2024 22:52:00 +1200 Subject: [PATCH 47/57] Fix 'labels' config parsing --- .../utils/dependabot-cli/DependabotOutputProcessor.ts | 2 +- .../utils/dependabot/interfaces/IDependabotConfig.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index ac5bf9e2..5084a6e7 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -117,7 +117,7 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess } : undefined, assignees: update.config.assignees, reviewers: update.config.reviewers, - labels: update.config.labels?.split(',').map((label) => label.trim()) || [], + labels: update.config.labels?.map((label) => label?.trim()) || [], workItems: update.config.milestone ? [update.config.milestone] : [], changes: getPullRequestChangedFilesForOutputData(data), properties: buildPullRequestProperties(update.job["package-manager"], dependencies) diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot/interfaces/IDependabotConfig.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot/interfaces/IDependabotConfig.ts index c4cb4ed5..e7ed0faf 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot/interfaces/IDependabotConfig.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot/interfaces/IDependabotConfig.ts @@ -40,7 +40,7 @@ export interface IDependabotUpdate { 'groups'?: Record, 'ignore'?: IDependabotIgnoreCondition[], 'insecure-external-code-execution'?: string, - 'labels'?: string, + 'labels': string[], 'milestone'?: string, 'open-pull-requests-limit'?: number, 'pull-request-branch-name'?: IDependabotPullRequestBranchName, From 2434018baf3f2a747128955b98692c0cb9f3ddc0 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sat, 21 Sep 2024 22:54:43 +1200 Subject: [PATCH 48/57] Implement "targetUpdateIds" task input option --- .../tasks/dependabot/dependabotV2/index.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index 578c929b..c3ae716d 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -2,6 +2,7 @@ import { which, setResult, TaskResult } from "azure-pipelines-task-lib/task" import { debug, warning, error } from "azure-pipelines-task-lib/task" import { DependabotCli } from './utils/dependabot-cli/DependabotCli'; import { AzureDevOpsWebApiClient } from "./utils/azure-devops/AzureDevOpsWebApiClient"; +import { IDependabotUpdate } from "./utils/dependabot/interfaces/IDependabotConfig"; import { DependabotOutputProcessor, parseProjectDependencyListProperty, parsePullRequestProperties } from "./utils/dependabot-cli/DependabotOutputProcessor"; import { DependabotJobBuilder } from "./utils/dependabot-cli/DependabotJobBuilder"; import parseDependabotConfigFile from './utils/dependabot/parseConfigFile'; @@ -53,9 +54,20 @@ async function run() { updaterImage: undefined // TODO: Add config for this? }; - // Loop through each 'update' block in dependabot.yaml and perform updates - await Promise.all(dependabotConfig.updates.map(async (update) => { - + // If update identifiers are specified, select them; otherwise handle all + let updates: IDependabotUpdate[] = []; + const targetIds = taskInputs.targetUpdateIds; + if (targetIds && targetIds.length > 0) { + for (const id of targetIds) { + updates.push(dependabotConfig.updates[id]); + } + } else { + updates = dependabotConfig.updates; + } + + // Loop through the [targeted] update blocks in dependabot.yaml and perform updates + await Promise.all(updates.map(async (update) => { + // Parse the last dependency list snapshot (if any) from the project properties. // This is required when doing a security-only update as dependabot requires the list of vulnerable dependencies to be updated. // Automatic discovery of vulnerable dependencies during a security-only update is not currently supported by dependabot-updater. From c1c3930c2bd83405b405d50e9ee87ba8fc174f7c Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sat, 21 Sep 2024 23:25:27 +1200 Subject: [PATCH 49/57] Fix error when using multiple update blocks in dependabot.yml with the same package manager --- extension/tasks/dependabot/dependabotV2/index.ts | 10 ++++++---- .../utils/dependabot-cli/DependabotJobBuilder.ts | 8 ++++---- .../dependabot-cli/DependabotOutputProcessor.ts | 16 ++++++++++------ 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index c3ae716d..5d28eaad 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -67,6 +67,7 @@ async function run() { // Loop through the [targeted] update blocks in dependabot.yaml and perform updates await Promise.all(updates.map(async (update) => { + const updateId = updates.indexOf(update).toString(); // Parse the last dependency list snapshot (if any) from the project properties. // This is required when doing a security-only update as dependabot requires the list of vulnerable dependencies to be updated. @@ -80,9 +81,10 @@ async function run() { // Parse the Dependabot metadata for the existing pull requests that are related to this update // Dependabot will use this to determine if we need to create new pull requests or update/close existing ones const existingPullRequests = parsePullRequestProperties(prAuthorActivePullRequests, update["package-ecosystem"]); + const existingPullRequestDependencies = Object.entries(existingPullRequests).map(([id, deps]) => deps); // Run an update job for "all dependencies"; this will create new pull requests for dependencies that need updating - const allDependenciesJob = DependabotJobBuilder.newUpdateAllJob(taskInputs, update, dependabotConfig.registries, dependencyList['dependencies'], existingPullRequests); + const allDependenciesJob = DependabotJobBuilder.newUpdateAllJob(taskInputs, updateId, update, dependabotConfig.registries, dependencyList['dependencies'], existingPullRequestDependencies); const allDependenciesUpdateOutputs = await dependabot.update(allDependenciesJob, dependabotUpdaterOptions); if (!allDependenciesUpdateOutputs || allDependenciesUpdateOutputs.filter(u => !u.success).length > 0) { allDependenciesUpdateOutputs.filter(u => !u.success).forEach(u => exception(u.error)); @@ -91,15 +93,15 @@ async function run() { // Run an update job for each existing pull request; this will resolve merge conflicts and close pull requests that are no longer needed if (!taskInputs.skipPullRequests) { - for (const pr of existingPullRequests) { - const updatePullRequestJob = DependabotJobBuilder.newUpdatePullRequestJob(taskInputs, update, dependabotConfig.registries, existingPullRequests, pr); + for (const pullRequestId in existingPullRequests) { + const updatePullRequestJob = DependabotJobBuilder.newUpdatePullRequestJob(taskInputs, pullRequestId, update, dependabotConfig.registries, existingPullRequestDependencies, existingPullRequests[pullRequestId]); const updatePullRequestOutputs = await dependabot.update(updatePullRequestJob, dependabotUpdaterOptions); if (!updatePullRequestOutputs || updatePullRequestOutputs.filter(u => !u.success).length > 0) { updatePullRequestOutputs.filter(u => !u.success).forEach(u => exception(u.error)); taskSucceeded = false; } } - } else if (existingPullRequests.length > 0) { + } else if (existingPullRequests.keys.length > 0) { warning(`Skipping update of existing pull requests as 'skipPullRequests' is set to 'true'`); return; } diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts index d6e9fb3b..d355c6bd 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts @@ -20,6 +20,7 @@ export class DependabotJobBuilder { */ public static newUpdateAllJob( taskInputs: ISharedVariables, + id: string, update: IDependabotUpdate, registries: Record, dependencyList: any[], @@ -29,7 +30,7 @@ export class DependabotJobBuilder { const securityUpdatesOnly = update["open-pull-requests-limit"] == 0; const updateDependencyNames = securityUpdatesOnly ? mapDependenciesForSecurityUpdate(dependencyList) : undefined; return buildUpdateJobConfig( - `update-${packageEcosystem}-${securityUpdatesOnly ? 'security-only' : 'all'}`, + `update-${id}-${packageEcosystem}-${securityUpdatesOnly ? 'security-only' : 'all'}`, taskInputs, update, registries, @@ -51,17 +52,16 @@ export class DependabotJobBuilder { */ public static newUpdatePullRequestJob( taskInputs: ISharedVariables, + id: string, update: IDependabotUpdate, registries: Record, existingPullRequests: any[], pullRequestToUpdate: any ): IDependabotUpdateOperation { - const packageEcosystem = update["package-ecosystem"]; const dependencyGroupName = pullRequestToUpdate['dependency-group-name']; const dependencies = (dependencyGroupName ? pullRequestToUpdate['dependencies'] : pullRequestToUpdate)?.map(d => d['dependency-name']); - const dependencyNamesHash = crypto.createHash('md5').update(dependencies.join(',')).digest('hex').substring(0, 10) return buildUpdateJobConfig( - `update-${packageEcosystem}-pr-${dependencyNamesHash}`, + `update-pr-${id}`, taskInputs, update, registries, diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index 5084a6e7..5fcbec18 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -249,16 +249,20 @@ export function parseProjectDependencyListProperty(properties: Record { + return Object.fromEntries(pullRequests .filter(pr => { return pr.properties.find(p => p.name === DependabotOutputProcessor.PR_PROPERTY_NAME_PACKAGE_MANAGER && (packageManager === null || p.value === packageManager)); }) .map(pr => { - return JSON.parse( - pr.properties.find(p => p.name === DependabotOutputProcessor.PR_PROPERTY_NAME_DEPENDENCIES)?.value - ) - }); + return [ + pr.id, + JSON.parse( + pr.properties.find(p => p.name === DependabotOutputProcessor.PR_PROPERTY_NAME_DEPENDENCIES)?.value + ) + ]; + }) + ); } function getSourceBranchNameForUpdate(packageEcosystem: string, targetBranch: string, dependencies: any): string { From 0803a30b8af71e796b05a9077b57cfd9cd1eea55 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sat, 21 Sep 2024 23:41:37 +1200 Subject: [PATCH 50/57] Only install dependabot once; cache the tool path once known --- .../utils/dependabot-cli/DependabotCli.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts index 4f213c15..5a044f89 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts @@ -19,6 +19,8 @@ export class DependabotCli { private readonly outputProcessor: IDependabotUpdateOutputProcessor; private readonly debug: boolean; + private toolPath: string; + public static readonly CLI_IMAGE_LATEST = "github.com/dependabot/cli/cmd/dependabot@latest"; constructor(cliToolImage: string, outputProcessor: IDependabotUpdateOutputProcessor, debug: boolean) { @@ -129,9 +131,9 @@ export class DependabotCli { private async getDependabotToolPath(installIfMissing: boolean = true): Promise { debug('Checking for `dependabot` install...'); - let dependabotPath = which("dependabot", false); - if (dependabotPath) { - return dependabotPath; + this.toolPath ||= which("dependabot", false); + if (this.toolPath) { + return this.toolPath; } if (!installIfMissing) { throw new Error("Dependabot CLI install not found"); @@ -142,10 +144,11 @@ export class DependabotCli { goTool.arg(["install", this.toolImage]); goTool.execSync(); - // Depending on how go is installed on the host agent, the go bin path may not be in the PATH environment variable. - // If `which("dependabot")` still doesn't resolve, we must manually resolve the path; It will either be "$GOPATH/bin/dependabot" or "$HOME/go/bin/dependabot" if $GOPATH is not set. + // Depending on how Go is configured on the host agent, the "go/bin" path may not be in the PATH environment variable. + // If dependabot still cannot be found using `which()` after install, we must manually resolve the path; + // It will either be "$GOPATH/bin/dependabot" or "$HOME/go/bin/dependabot", if GOPATH is not set. const goBinPath = process.env.GOPATH ? path.join(process.env.GOPATH, 'bin') : path.join(os.homedir(), 'go', 'bin'); - return which("dependabot", false) || path.join(goBinPath, 'dependabot'); + return this.toolPath ||= which("dependabot", false) || path.join(goBinPath, 'dependabot'); } // Create the jobs directory if it does not exist From 47998c7ef94ae6787dd19a8878a774345784bbe1 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sat, 21 Sep 2024 23:52:07 +1200 Subject: [PATCH 51/57] Add migration warning to complete V1 pull requests before migrating to V2 --- docs/migrations/v1-to-v2.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/migrations/v1-to-v2.md b/docs/migrations/v1-to-v2.md index 962b5531..35cfd7ef 100644 --- a/docs/migrations/v1-to-v2.md +++ b/docs/migrations/v1-to-v2.md @@ -17,6 +17,9 @@ The task now uses [Dependabot CLI](https://github.com/dependabot/cli) to perform # Breaking changes V1 → V2 +> [!WARNING] +> **It is strongly recommended that you complete (or abandon) all active Depedabot pull requests created in V1 before migrating to V2.** Due to changes in Dependabot dependency metadata, V2 pull requests are not compatible with V1 (and vice versa). Migrating to V2 before completing existing pull requests will lead to duplication of pull requests. + ### New pipeline agent requirements; "Go" must be installed Dependabot CLI requires [Go](https://go.dev/doc/install) (1.22+) and [Docker](https://docs.docker.com/get-started/get-docker/) (with Linux containers). If you use [Microsoft-hosted agents](https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops&tabs=yaml#software), we recommend using the [ubuntu-latest](https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md) image, which meets all task requirements. From bbba6842a72207bdb13f9187d555dbb2fd4726b5 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sun, 22 Sep 2024 00:23:36 +1200 Subject: [PATCH 52/57] Process updates synchronously when using multiple update blocks in dependabot.yml --- extension/tasks/dependabot/dependabotV2/index.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index 5d28eaad..8432645b 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -66,7 +66,7 @@ async function run() { } // Loop through the [targeted] update blocks in dependabot.yaml and perform updates - await Promise.all(updates.map(async (update) => { + for (const update of updates) { const updateId = updates.indexOf(update).toString(); // Parse the last dependency list snapshot (if any) from the project properties. @@ -103,10 +103,9 @@ async function run() { } } else if (existingPullRequests.keys.length > 0) { warning(`Skipping update of existing pull requests as 'skipPullRequests' is set to 'true'`); - return; } - })); + } setResult( taskSucceeded ? TaskResult.Succeeded : TaskResult.Failed, From affe505f0b87b903fd4db6bff5d269a14f97a3fe Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sun, 22 Sep 2024 00:23:53 +1200 Subject: [PATCH 53/57] Fix typos --- .../dependabotV2/utils/dependabot-cli/DependabotCli.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts index 5a044f89..f3781689 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotCli.ts @@ -80,7 +80,7 @@ export class DependabotCli { // Run dependabot update if (!fs.existsSync(jobOutputPath) || fs.statSync(jobOutputPath)?.size == 0) { - console.info(`Running Dependabot update job from '${jobInputPath}'...`); + console.info(`Running Dependabot update job '${jobInputPath}'...`); const dependabotTool = tool(dependabotPath).arg(dependabotArguments); const dependabotResultCode = await dependabotTool.execAsync({ failOnStdErr: false, @@ -96,7 +96,7 @@ export class DependabotCli { if (fs.existsSync(jobOutputPath)) { const jobOutputs = readJobScenarioOutputFile(jobOutputPath); if (jobOutputs?.length > 0) { - console.info(`Processing Dependabot outputs from '${jobInputPath}'...`); + console.info(`Processing outputs from '${jobOutputPath}'...`); for (const output of jobOutputs) { // Documentation on the scenario model can be found here: // https://github.com/dependabot/cli/blob/main/internal/model/scenario.go From 5978e761063489cbcc1410beb3b5ede80d94da94 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sun, 22 Sep 2024 00:29:33 +1200 Subject: [PATCH 54/57] Report the total number of failed update jobs in the task result --- extension/tasks/dependabot/dependabotV2/index.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index 8432645b..8bb59247 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -9,8 +9,8 @@ import parseDependabotConfigFile from './utils/dependabot/parseConfigFile'; import parseTaskInputConfiguration from './utils/getSharedVariables'; async function run() { - let taskSucceeded: boolean = true; let dependabot: DependabotCli = undefined; + let failedJobs: number = 0; try { // Check if required tools are installed @@ -88,7 +88,7 @@ async function run() { const allDependenciesUpdateOutputs = await dependabot.update(allDependenciesJob, dependabotUpdaterOptions); if (!allDependenciesUpdateOutputs || allDependenciesUpdateOutputs.filter(u => !u.success).length > 0) { allDependenciesUpdateOutputs.filter(u => !u.success).forEach(u => exception(u.error)); - taskSucceeded = false; + failedJobs++; } // Run an update job for each existing pull request; this will resolve merge conflicts and close pull requests that are no longer needed @@ -98,7 +98,7 @@ async function run() { const updatePullRequestOutputs = await dependabot.update(updatePullRequestJob, dependabotUpdaterOptions); if (!updatePullRequestOutputs || updatePullRequestOutputs.filter(u => !u.success).length > 0) { updatePullRequestOutputs.filter(u => !u.success).forEach(u => exception(u.error)); - taskSucceeded = false; + failedJobs++; } } } else if (existingPullRequests.keys.length > 0) { @@ -108,8 +108,8 @@ async function run() { } setResult( - taskSucceeded ? TaskResult.Succeeded : TaskResult.Failed, - taskSucceeded ? 'All update jobs completed successfully' : 'One or more update jobs failed, check logs for more information' + failedJobs ? TaskResult.Failed : TaskResult.Succeeded, + failedJobs ? `${failedJobs} update job(s) failed, check logs for more information` : `All update jobs completed successfully` ); } From 94f654a99275902dd881ebae65c6f1a8a97e0c72 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sun, 22 Sep 2024 01:11:36 +1200 Subject: [PATCH 55/57] Include stack trace when errors are logged, to help with diagnosing issues --- .../tasks/dependabot/dependabotV2/index.ts | 5 ++--- .../azure-devops/AzureDevOpsWebApiClient.ts | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index 8bb59247..34ea0369 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -123,9 +123,8 @@ async function run() { } function exception(e: Error) { - if (e?.stack) { - error(e.stack); - } + error(`An unhandled exception occurred: ${e}`); + console.error(e); } run(); diff --git a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts index 1d8b06cc..a7288517 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts @@ -55,7 +55,7 @@ export class AzureDevOpsWebApiClient { * @param repository * @returns */ - public async getDefaultBranch(project: string, repository: string): Promise { + public async getDefaultBranch(project: string, repository: string): Promise { try { const git = await this.connection.getGitApi(); const repo = await git.getRepository(repository, project); @@ -67,7 +67,8 @@ export class AzureDevOpsWebApiClient { } catch (e) { error(`Failed to get default branch for '${project}/${repository}': ${e}`); - throw e; + console.error(e); + return undefined; } } @@ -108,6 +109,7 @@ export class AzureDevOpsWebApiClient { } catch (e) { error(`Failed to list active pull request properties: ${e}`); + console.error(e); return []; } } @@ -256,6 +258,7 @@ export class AzureDevOpsWebApiClient { } catch (e) { error(`Failed to create pull request: ${e}`); + console.error(e); return null; } } @@ -336,6 +339,7 @@ export class AzureDevOpsWebApiClient { } catch (e) { error(`Failed to update pull request: ${e}`); + console.error(e); return false; } } @@ -372,6 +376,7 @@ export class AzureDevOpsWebApiClient { } catch (e) { error(`Failed to approve pull request: ${e}`); + console.error(e); return false; } } @@ -450,6 +455,7 @@ export class AzureDevOpsWebApiClient { } catch (e) { error(`Failed to close pull request: ${e}`); + console.error(e); return false; } } @@ -470,8 +476,11 @@ export class AzureDevOpsWebApiClient { .map(p => ({ [p.name]: p.value })) .reduce((a, b) => ({ ...a, ...b }), {}); - } catch (e) { + } + catch (e) { error(`Failed to get project properties: ${e}`); + console.error(e); + return undefined; } } @@ -505,8 +514,10 @@ export class AzureDevOpsWebApiClient { ] ); - } catch (e) { + } + catch (e) { error(`Failed to update project property '${name}': ${e}`); + console.error(e); } } } From 5450d1d9b2c292f3629de76675dcb1068e1decf5 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sun, 22 Sep 2024 01:21:08 +1200 Subject: [PATCH 56/57] Fix inverted logic for "abandonUnwantedPullRequests" --- .../utils/dependabot-cli/DependabotOutputProcessor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts index 5fcbec18..8e1a9308 100644 --- a/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts +++ b/extension/tasks/dependabot/dependabotV2/utils/dependabot-cli/DependabotOutputProcessor.ts @@ -169,8 +169,8 @@ export class DependabotOutputProcessor implements IDependabotUpdateOutputProcess return pullRequestWasUpdated; case 'close_pull_request': - if (this.taskInputs.abandonUnwantedPullRequests) { - warning(`Skipping pull request closure as 'abandonUnwantedPullRequests' is set to 'true'`); + if (!this.taskInputs.abandonUnwantedPullRequests) { + warning(`Skipping pull request closure as 'abandonUnwantedPullRequests' is set to 'false'`); return true; } From a9f6deb993fc5bad073d4f666789425dbd6875a5 Mon Sep 17 00:00:00 2001 From: Rhys Koedijk Date: Sun, 22 Sep 2024 01:38:59 +1200 Subject: [PATCH 57/57] Fix error handling --- extension/tasks/dependabot/dependabotV2/index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/extension/tasks/dependabot/dependabotV2/index.ts b/extension/tasks/dependabot/dependabotV2/index.ts index 34ea0369..b899edf7 100644 --- a/extension/tasks/dependabot/dependabotV2/index.ts +++ b/extension/tasks/dependabot/dependabotV2/index.ts @@ -123,8 +123,10 @@ async function run() { } function exception(e: Error) { - error(`An unhandled exception occurred: ${e}`); - console.error(e); + if (e) { + error(`An unhandled exception occurred: ${e}`); + console.error(e); + } } run();