Merged master into current branch.

.gitlab/ci/.gitlab-ci.global.yml

@@ -224,23 +224,21 @@
   - section_end "Gitlab CI Build Parameters"
 
 .checkout-upload-commit-content-nightly: &checkout-upload-commit-content-nightly
-  - section_start "Checkout upload commit content nightly"
+  - section_start "Checkout upload commit content nightly" --collapsed
   - | 
     if [[ -n "${NIGHTLY}" && "${CI_COMMIT_BRANCH}" == "master" ]]; then
-      echo "content nightly: checking out to the last successful upload commit $LAST_UPLOAD_COMMIT for the Packs folder"
-      git status
-      git reset $LAST_UPLOAD_COMMIT -- ./Packs  # resets the state of the "Packs" folder to the last successful upload commit, it discards any changes made to the folder since that commit
-      git checkout -- ./Packs  # checks out the "Packs" folder to the state of the last successful upload commit. it ensures that the folder matches the commit's content.
-      git clean -fd ./Packs  # cleans any untracked files and directories in the "Packs" folder. it ensures that only tracked files remain.
-      deleted_files=$(git diff --name-only --diff-filter=D $LAST_UPLOAD_COMMIT..HEAD -- Packs)  # get all the files deleted between the last successful upload commit to the master
-      if [[ -n "${deleted_files}" ]]; then
-        echo "deleted files between last upload commit $LAST_UPLOAD_COMMIT to branch $(git branch --show-current) are $deleted_files"
+      if [[ ! -d "${CI_PROJECT_DIR}/artifacts/production_packs" ]]; then
+        echo "content production packs do not exist in ${CI_PROJECT_DIR}/artifacts"
+        exit 1
       fi
-      for file in $deleted_files; do
-        git checkout HEAD -- "$file"  # restore any deleted files between the master branch and the last successful upload commit.
-      done
-      git status
-      echo "content nightly: successfully reverted the Packs folder to upload commit $LAST_UPLOAD_COMMIT"
+      rm -rf ./Packs
+      echo "copying production Packs folder from ${CI_PROJECT_DIR}/artifacts/production_packs to ./Packs"
+      cp -r ${CI_PROJECT_DIR}/artifacts/production_packs ./Packs
+      git config core.fileMode false  # used to tell git not to identify permission changes on files as changes
+      chmod -R 777 ./Packs  # required for the lint, we use the permissions of the file when running pytest within the containers
+      echo "the Packs changes between upload commit ${LAST_UPLOAD_COMMIT} to master commit ${CI_COMMIT_SHA} is:"
+      git status -- Packs  # show the differences between the upload commit to the master branch for the Packs folder
+      echo "The Packs folder is in the state of commit $LAST_UPLOAD_COMMIT"
     else
       echo "not checking out to the latest upload commit $LAST_UPLOAD_COMMIT because the build is not content nightly"
     fi
@@ -274,11 +272,28 @@
   - *install_node_modules
   - *install_venv
 
+.add-content-production-to-artifacts:
+  - section_start "Clone production content and add it to artifacts" --collapsed
+  - mkdir content_production
+  - cd content_production
+  - git init > /dev/null 2>&1
+  - git remote add origin https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_NAMESPACE}/content.git
+  - git fetch --depth 1 origin $LAST_UPLOAD_COMMIT
+  - git checkout FETCH_HEAD >${ARTIFACTS_FOLDER}/logs/add-content-production-to-artifacts.log 2>&1
+  - cp -r ./Packs ${ARTIFACTS_FOLDER}/production_packs
+  - echo "checked out ${LAST_UPLOAD_COMMIT} which is the last successful upload commit"
+  - section_end "Clone production content and add it to artifacts"
+  - job-done
+
+
 .default-job-settings:
   interruptible: true
   extends:
     - .default-cache
     - .default-before-script
+  needs:
+    - job: cloning-content-repo-last-upload-commit
+      optional: true
 
 .trigger-slack-notification:
   stage: .post
@@ -362,7 +377,6 @@
 .unittests-and-lint-settings:
   tags:
     - gce
-  needs: [ ]
   stage: unittests-and-validations
   artifacts:
     reports:
@@ -459,7 +473,6 @@
   stage: unittests-and-validations
   extends:
     - .default-job-settings
-  needs: []
   variables:
     KUBERNETES_CPU_REQUEST: 1000m
   artifacts:
@@ -514,3 +527,23 @@
     - .default-job-settings
   script:
     - python3 Tests/scripts/check_jobs_done.py --triggering-workflow "${WORKFLOW}" --job-done-files "${PIPELINE_JOBS_FOLDER}"
+
+cloning-content-repo-last-upload-commit:
+  stage: .pre
+  rules:
+    - if: '$NIGHTLY'
+  artifacts:
+    expire_in: 30 days
+    paths:
+      - ${CI_PROJECT_DIR}/artifacts/*
+      - ${CI_PROJECT_DIR}/pipeline_jobs_folder/*
+    when: always
+  before_script:
+    - source .gitlab/helper_functions.sh
+    - *setup-network-certs
+    - *create_artifacts_and_server_type_instance_folders
+    - *get_last_upload_commit
+  variables:
+    ARTIFACTS_FOLDER: ${CI_PROJECT_DIR}/artifacts
+  script:
+    - !reference [.add-content-production-to-artifacts]

.gitlab/ci/.gitlab-ci.on-push.yml

@@ -12,7 +12,6 @@
 trigger-private-build:
   tags:
     - gke
-  needs: []
   stage: unittests-and-validations
   extends:
     - .default-job-settings
@@ -67,7 +66,6 @@ stop-running-pipelines:
   tags:
     - gke
   stage: unittests-and-validations
-  needs: []
   extends:
     - .default-job-settings
   variables:
@@ -112,7 +110,6 @@ validate-content-conf:
   tags:
     - gke
   stage: unittests-and-validations
-  needs: []
   extends:
     - .default-job-settings
   rules:
@@ -135,7 +132,6 @@ validate-content-conf:
   variables:
     KUBERNETES_CPU_REQUEST: 2000m
     EXTRACT_PRIVATE_TESTDATA: "true"
-  needs: []
   stage: prepare-testing-bucket
   script:
     - !reference [.download-demisto-conf]
@@ -282,7 +278,6 @@ xpanse-prepare-testing-bucket:
   stage: run-instances
   needs:
     - job: xsoar-prepare-testing-bucket
-      optional: true
   script:
     - EXIT_CODE=0
     - section_start "Check if should run Instance role"
@@ -378,6 +373,7 @@ jobs-done-check-nightly:
   extends:
     - .jobs-done-check
   needs:
+    - cloning-content-repo-last-upload-commit
     - run-unittests-and-lint
     - run-validations
     - trigger-private-build

Packs/AzureSentinel/Integrations/AzureSentinel/AzureSentinel.py

@@ -1250,7 +1250,7 @@ def fetch_incidents(client: AzureSentinelClient, last_run: dict, first_fetch_tim
     last_incident_number = last_run.get('last_incident_number')
     demisto.debug(f"{last_fetch_time=}, {last_fetch_ids=}, {last_incident_number=}")
 
-    if last_fetch_time is None or last_incident_number is None:
+    if last_fetch_time is None or not last_incident_number:
         demisto.debug("handle via timestamp")
         if last_fetch_time is None:
             last_fetch_time_str, _ = parse_date_range(first_fetch_time, DATE_FORMAT)
@@ -1346,7 +1346,7 @@ def process_incidents(raw_incidents: list, min_severity: int, latest_created_tim
             incidents.append(xsoar_incident)
         else:
             demisto.debug(f"drop creation of {incident.get('IncidentNumber')=} "
-                          "due to the {incident_severity=} is lower then {min_severity=}")
+                          f"due to the {incident_severity=} is lower then {min_severity=}")
 
         # Update last run to the latest fetch time
         if incident_created_time is None:
@@ -1356,7 +1356,6 @@ def process_incidents(raw_incidents: list, min_severity: int, latest_created_tim
             latest_created_time = incident_created_time
         if incident.get('IncidentNumber') > last_incident_number:
             last_incident_number = incident.get('IncidentNumber')
-
     next_run = {
         'last_fetch_time': latest_created_time.strftime(DATE_FORMAT),
         'last_fetch_ids': current_fetch_ids,

Packs/AzureSentinel/Integrations/AzureSentinel/AzureSentinel.yml

@@ -2360,7 +2360,7 @@ script:
     execution: false
     name: azure-sentinel-auth-reset
     arguments: []
-  dockerimage: demisto/crypto:1.0.0.80214
+  dockerimage: demisto/crypto:1.0.0.80694
   isfetch: true
   runonce: false
   script: '-'

Packs/AzureSentinel/ReleaseNotes/1_5_29.md

@@ -0,0 +1,6 @@
+
+#### Integrations
+
+##### Microsoft Sentinel
+
+- Fixed an issue where the `First fetch timestamp` argument was not respected in **fetch-incidents**.

Packs/AzureSentinel/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Microsoft Sentinel",
     "description": "Microsoft Sentinel is a cloud-native security information and event manager (SIEM) platform that uses built-in AI to help analyze large volumes of data across an enterprise.",
     "support": "xsoar",
-    "currentVersion": "1.5.28",
+    "currentVersion": "1.5.29",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

Packs/CommonScripts/ReleaseNotes/1_12_47.md

@@ -1,6 +1,6 @@
 
 #### Scripts
 
-##### DBotClosedIncidentsPercentage
+##### IsIntegrationAvailable
 
-Updated the query in the**DBotClosedIncidentsPercentage** script to `status:closed and closingUser:DBot`.
+- Fixed an issue where the *outputs* for *Conditional* Tasks were not 'yes' or 'no' (issue was introduced in 1.12.46).

Packs/CommonScripts/Scripts/IsIntegrationAvailable/IsIntegrationAvailable.py

@@ -11,7 +11,8 @@ def is_integration_available(brand_name: str, all_instances: Dict[str, dict]) ->
     return CommandResults(
         outputs_prefix='brandInstances',
         outputs=brand_instances,
-        readable_output=readable_output
+        readable_output=readable_output,
+        raw_response=readable_output
     )
 
 

Packs/CommonScripts/Scripts/IsIntegrationAvailable/IsIntegrationAvailable.yml

@@ -20,9 +20,10 @@ scripttarget: 0
 dependson: {}
 timeout: 0s
 outputs:
-- contextPath: brandInstances
-  description: List of the instances for the given brands.
-  type: Unknown
+- contextPath: yes
+  description: if integration is available.
+- contextPath: no
+  description: if there is no instance for the given brand.
 fromversion: 5.0.0
 dockerimage: demisto/python3:3.10.13.80593
 tests:

Packs/FortiGate/ParsingRules/FortiGate/FortiGate.xif

@@ -1,7 +1,10 @@
 [INGEST:vendor="Fortinet", product="Fortigate", target_dataset="fortinet_fortigate_raw", no_hit=keep]
-filter FTNTFGTeventtime ~= "\d{9}$"
+filter to_string(FTNTFGTeventtime) ~= "\d{9}$"
 | alter
-    tmp_remove_milli_secs = to_integer(replex(FTNTFGTeventtime, "\d{9}$", ""))
+    tmp_remove_milli_secs = to_integer(replex(FTNTFGTeventtime, "\d{9}$", "")),
+    tmp_check_duration = if(FTNTFGTduration = null, 0, to_integer(FTNTFGTduration))
 | alter
-    _time = to_timestamp(tmp_remove_milli_secs)
-| fields -tmp_remove_milli_secs;
+    tmp_substract_diff = subtract(tmp_remove_milli_secs, tmp_check_duration)
+| alter
+    _time = to_timestamp(tmp_substract_diff)
+| fields -tmp_remove_milli_secs, tmp_check_duration, tmp_substract_diff;

Packs/FortiGate/README.md

@@ -23,7 +23,9 @@ More information can be found [here](https://docs.fortinet.com/document/fortigat
 ## Collect Events from Vendor
 In order to use the collector, use the [Broker VM](#broker-vm) option.
 
-* Support for timestamp parsing is available only for the **FTNTFGTeventtime** field in UTC +0000 time zone.
+### Timestamp Ingestion:
+* Support for timestamp parsing is available only for the **FTNTFGTeventtime** and **FTNTFGTduration** fields in Epoch (UTC) format.
+* Timestamp calculation for logs is the presented result of deducting **FTNTFGTduration** from **FTNTFGTeventtime**.
 
 ### Broker VM
 You will need to use the information described [here](https://docs-cortex.paloaltonetworks.com/r/Cortex-XDR/Cortex-XDR-Pro-Administrator-Guide/Configure-the-Broker-VM).\

Packs/FortiGate/ReleaseNotes/1_0_26.md

@@ -0,0 +1,6 @@
+
+#### Parsing Rules
+
+##### FortiGate
+
+Updated the Parsing Rule logic, deducting the FTNTFGTduration value from FTNTFGTeventtime.

Packs/FortiGate/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "FortiGate",
     "description": "Manage FortiGate Firewall",
     "support": "xsoar",
-    "currentVersion": "1.0.25",
+    "currentVersion": "1.0.26",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

Packs/VectraXDR/.pack-ignore

@@ -0,0 +1,86 @@
+[file:incidentfield-Vectra-XDR-Entity-State.json]
+ignore=IF115
+
+[file:incidentfield-Vectra-XDR-Entity-ID.json]
+ignore=IF115
+
+[file:incidentfield-Vectra-XDR-Entity-Attack-Rating.json]
+ignore=IF115
+
+[file:incidentfield-Vectra-XDR-Entity-Host-Type.json]
+ignore=IF115
+
+[file:incidentfield-Vectra-XDR-Entity-Priority-Status.json]
+ignore=IF115
+
+[file:incidentfield-Vectra-XDR-Entity-Urgency-Score.json]
+ignore=IF115
+
+[file:incidentfield-Vectra-XDR-Entity-Attack-Profile.json]
+ignore=IF115
+
+[file:incidentfield-Vectra-XDR-Entity-Last-Detection-Timestamp.json]
+ignore=IF115
+
+[file:incidentfield-Vectra-XDR-Entity-Last-Modification-Timestamp.json]
+ignore=IF115
+
+[file:incidentfield-Vectra-XDR-Entity-Type.json]
+ignore=IF115
+
+[file:incidentfield-Vectra-XDR-Entity-Name.json]
+ignore=IF115
+
+[file:incidentfield-Vectra-XDR-Entity-Account-Type.json]
+ignore=IF115
+
+[known_words]
+vectra
+Unassign
+Unmark
+s
+interracting
+br
+d
+utc
+mdr
+ds
+n
+cnc
+t
+r
+stratus
+HashiCorp
+rpc
+Certainnity
+builtin
+ss
+ng
+dc
+Oasuc
+rx
+ox
+dg
+Prioritization
+dropdown
+mappings
+sts
+sn
+x
+eyu
+yng
+pj
+nasx
+fyey
+pm
+k
+hfcj
+assignees
+Cognito
+ipam
+ps
+accout
+knwoledge
+Entit
+Makr
+Ingration

Packs/VectraXDR/.secrets-ignore

@@ -0,0 +1,20 @@
+0.0.0.4
+test.user4@mail.com
+0.0.0.3
+0.0.0.1
+0.0.0.2
+test_user2@gmail.com
+192.168.30.189
+VectraXDRAddNotesInLayout
+?pivot=Vectra-XSOAR-1.0.0
+::1936
+https://www.terraform.io
+HashiCorp-terraform-exec/0.17.3
+test_user@mail.com
+709db6e1f8f5054ca57caf43ba248ed6
+IP-192.168.55.10_hidden_dns_tunnel_1382.pcap
+384:dN+Pm11R0XPmts64kZog9ZaikYngk+SnRxFyeyCEyuAOasucOcakca0/rHfcjOUI:dI+t25caEPjRSnmuNasxRana4DgOUDcX
+**bold**\n\n_Italic_\n\n+Underline+\n\n~~strikethrough~~\n\nAdded
+test.user2@mail.com
+test.user3@mail.com
+test.user@mail.com