Skip to content

Commit

Permalink
Merge branch 'google/2.6' into wangdi/google_dfuse_fix
Browse files Browse the repository at this point in the history
Signed-off-by: Jeff Olivier <jeffolivier@google.com>
  • Loading branch information
jolivier23 authored Jan 13, 2025
2 parents 3b7ca05 + 0c8f5f6 commit 09dea61
Show file tree
Hide file tree
Showing 6 changed files with 254 additions and 144 deletions.
78 changes: 18 additions & 60 deletions src/client/dfuse/dfuse_core.c
Original file line number Diff line number Diff line change
Expand Up @@ -78,11 +78,11 @@ dfuse_progress_thread(void *arg)
static int
dfuse_parse_time(char *buff, size_t len, unsigned int *_out)
{
int matched;
unsigned int out = 0;
int count0 = 0;
int count1 = 0;
char c = '\0';
int matched;
unsigned int out = 0;
int count0 = 0;
int count1 = 0;
char c = '\0';

matched = sscanf(buff, "%u%n%c%n", &out, &count0, &c, &count1);

Expand Down Expand Up @@ -536,7 +536,7 @@ dfuse_pool_connect(struct dfuse_info *dfuse_info, const char *label, struct dfus
err_disconnect:
ret = daos_pool_disconnect(dfp->dfp_poh, NULL);
if (ret)
DFUSE_TRA_WARNING(dfp, "Failed to disconnect pool: "DF_RC, DP_RC(ret));
DFUSE_TRA_WARNING(dfp, "Failed to disconnect pool: " DF_RC, DP_RC(ret));
err_free:
D_FREE(dfp);
err:
Expand Down Expand Up @@ -1375,20 +1375,20 @@ dfuse_event_release(void *arg)
int
dfuse_fs_start(struct dfuse_info *dfuse_info, struct dfuse_cont *dfs)
{
struct fuse_args args = {0};
struct fuse_args args = {0};
struct dfuse_inode_entry *ie;
struct d_slab_reg read_slab = {.sr_init = dfuse_event_init,
.sr_reset = dfuse_read_event_reset,
.sr_release = dfuse_event_release,
POOL_TYPE_INIT(dfuse_event, de_list)};
struct d_slab_reg read_slab = {.sr_init = dfuse_event_init,
.sr_reset = dfuse_read_event_reset,
.sr_release = dfuse_event_release,
POOL_TYPE_INIT(dfuse_event, de_list)};
struct d_slab_reg pre_read_slab = {.sr_init = dfuse_event_init,
.sr_reset = dfuse_pre_read_event_reset,
.sr_release = dfuse_event_release,
POOL_TYPE_INIT(dfuse_event, de_list)};
struct d_slab_reg write_slab = {.sr_init = dfuse_event_init,
.sr_reset = dfuse_write_event_reset,
.sr_release = dfuse_event_release,
POOL_TYPE_INIT(dfuse_event, de_list)};
struct d_slab_reg write_slab = {.sr_init = dfuse_event_init,
.sr_reset = dfuse_write_event_reset,
.sr_release = dfuse_event_release,
POOL_TYPE_INIT(dfuse_event, de_list)};
int rc;
int idx = 0;

Expand Down Expand Up @@ -1578,41 +1578,6 @@ ino_dfs_flush_nr(d_list_t *rlink, void *arg)
return -DER_SUCCESS;
}

static int
ino_kernel_flush(d_list_t *rlink, void *arg)
{
struct dfuse_info *dfuse_info = arg;
struct dfuse_inode_entry *ie = container_of(rlink, struct dfuse_inode_entry, ie_htl);
int rc;

/* Only evict entries that are direct children of the root, the kernel
* will walk the tree for us
*/
if (ie->ie_parent != 1)
return 0;

/* Do not evict root itself */
if (ie->ie_stat.st_ino == 1)
return 0;

rc = fuse_lowlevel_notify_inval_entry(dfuse_info->di_session, ie->ie_parent, ie->ie_name,
strlen(ie->ie_name));
if (rc != 0 && rc != -EBADF)
DHS_WARN(ie, -rc, "%#lx %#lx " DF_DE, ie->ie_parent, ie->ie_stat.st_ino,
DP_DE(ie->ie_name));
else
DHS_INFO(ie, -rc, "%#lx %#lx " DF_DE, ie->ie_parent, ie->ie_stat.st_ino,
DP_DE(ie->ie_name));

/* If the FUSE connection is dead then do not traverse further, it
* doesn't matter what gets returned here, as long as it's negative
*/
if (rc == -EBADF)
return -DER_NO_HDL;

return -DER_SUCCESS;
}

static int
dfuse_cont_close_cb(d_list_t *rlink, void *handle)
{
Expand All @@ -1636,10 +1601,10 @@ dfuse_cont_close_cb(d_list_t *rlink, void *handle)
static int
dfuse_pool_close_cb(d_list_t *rlink, void *handle)
{
struct dfuse_info *dfuse_info = handle;
struct dfuse_info *dfuse_info = handle;
struct dfuse_cont_core *dfcc, *dfccn;
struct dfuse_pool *dfp;
int rc;
struct dfuse_pool *dfp;
int rc;

dfp = container_of(rlink, struct dfuse_pool, dfp_entry);

Expand Down Expand Up @@ -1701,13 +1666,6 @@ dfuse_fs_stop(struct dfuse_info *dfuse_info)
sem_destroy(&eqt->de_sem);
}

/* First flush, instruct the kernel to forget items. This will run and work in ideal cases
* but often if the filesystem is unmounted it'll abort part-way through.
*/
rc = d_hash_table_traverse(&dfuse_info->dpi_iet, ino_kernel_flush, dfuse_info);

DHL_INFO(dfuse_info, rc, "Kernel flush complete");

/* At this point there's a number of inodes which are in memory, traverse these and free
* them, along with any resources.
* The reference count on inodes match kernel references but the fuse module is disconnected
Expand Down
217 changes: 217 additions & 0 deletions utils/cq/check_update_copyright.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
#!/bin/bash
#
# Copyright 2024 Intel Corporation.
# Copyright 2025 Hewlett Packard Enterprise Development LP
# Copyright 2025 Google LLC
#
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
# Check or update copyright date in modified files.
# Usage: check_update_copyright.sh <git_target> <githook|gha>
# mode "githook" will update copyright dates in place.
# mode "gha" will just print a warning in a GHA-compatible format.

set -e

git_target="$1"
mode="$2"
case "$mode" in
"githook" | "gha")
;;
*)
echo "Usage: check_update_copyright.sh <githook|gha>"
exit 1
esac

# Navigate to repo root
PARENT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
cd "$PARENT_DIR"/../../


regex_intel='(^[[:blank:]]*[\*/]*.*)((Copyright[[:blank:]]*)([0-9]{4})(-([0-9]{4}))?)([[:blank:]]*(Intel.*$))'
shortname_intel="Intel Corporation."
regex_hpe='(^[[:blank:]]*[\*/]*.*)((Copyright[[:blank:]]*)([0-9]{4})(-([0-9]{4}))?)([[:blank:]]*(Hewlett Packard Enterprise Development LP.*$))'
shortname_hpe="Hewlett Packard Enterprise Development LP"
regex_google='(^[[:blank:]]*[\*/]*.*)((Copyright[[:blank:]]*)([0-9]{4})(-([0-9]{4}))?)([[:blank:]]*(Google LLC.*$))'
shortname_google="Google LLC"
year=$(date +%Y)
errors=0
targets=(
# Entries with wildcard. These must be first and start with '*' or
# older versions of git will return files that were not changed.
'*.c'
'*.h'
'*.go'
'*.py'
'*.proto'
'*.java'
'*.yml'
'*.yaml'
'*.sh'
'*.bash'
'*Dockerfile*'
'*README*'
'*LICENSE*'
'*NOTICE*'
'*.txt'
'*.md'
# Entries without a wildcard
'Makefile'
'Jenkinsfile'
'SConscript'
'SConstruct'
'copyright'
'.env'
)

function git_reset() {
local file="$1"
if ! git reset "$file"; then
echo " Unable to un-stage $file"
errors=$((errors + 1))
return 1
fi
return 0
}

function git_add() {
local file="$1"
if ! git add "$file"; then
echo " Unable to re-stage $file"
errors=$((errors + 1))
return 1
fi
return 0
}


# Use HPE copyright for all users
# See below example to toggle copyright regex based on user
regex_user="$regex_hpe"
shortname_user="$shortname_hpe"
if [[ "$mode" == "githook" ]]; then
# Extract domain from configured email
user_domain="$(git config user.email | sed -n 's/^.*@\([-0-9a-zA-Z]*\).*/\1/p')"
else
# Extract domain from the first Signed-off-by
user_domain="$(git log -1 | grep 'Signed-off-by' | head -n 1 | sed -n 's/^.*@\([-0-9a-zA-Z]*\).*/\1/p')"
fi
case "$user_domain" in
"hpe")
regex_user="$regex_hpe"
shortname_user="$shortname_hpe"
;;
"intel")
regex_user="$regex_intel"
shortname_user="$shortname_intel"
;;
"google")
regex_user="$regex_google"
shortname_user="$shortname_google"
;;
*)
regex_user="$regex_hpe"
shortname_user="$shortname_hpe"
;;
esac

# Generate list of all copyright regex except the user's domain.
# Used to add a new copyright header to files.
all_regex_except_user=()
for _regex in "$regex_intel" "$regex_hpe"; do
if [[ "$_regex" != "$regex_user" ]]; then
all_regex_except_user+=("$_regex")
fi
done


if [ -z "$files" ]; then
files=$(git diff "$git_target" --cached --diff-filter=AM --name-only -- "${targets[@]}")
else
echo " Checking against custom files"
fi

os=$(uname -s)

. utils/githooks/git-version.sh

for file in $files; do
if [[ "$file" == *vendor* ]] || [[ "$file" == *pb.go ]] ||
[[ "$file" == *_string.go ]] || [[ "$file" == *pb-c* ]] ||
{ [ "$mode" == "githook" ] &&
[ "$git_vercode" -ge 2030000 ] &&
[ "$(git diff --cached -I Copyright "$file")" = '' ]; }; then
continue
fi

# Check for existing copyright in user's domain
# If it exists and is updated, nothing to do
read -r y1_user y2_user <<< "$(sed -nre "s/^.*$regex_user.*$/\4 \6/p" "$file")"
if [[ $y1_user -eq $year ]] || [[ $y2_user -eq $year ]]; then
continue
fi

# If user's domain copyright exists but is outdated, it needs to be updated
if [[ -n $y1_user ]] ; then
if [[ "$mode" == "githook" ]]; then
# Update copyright in place
git_reset "$file" || continue
if [[ "$os" == 'Linux' ]]; then
sed -i -re "s/$regex_user/\1Copyright $y1_user-$year \8/" "$file"
else
sed -i '' -re "s/$regex_user/\1Copyright $y1_user-$year \8/" "$file"
fi
git_add "$file" || continue
elif [[ "$mode" == "gha" ]]; then
# Print error but do not update
lineno="$(grep -nE "$regex_user" "$file" | cut -f1 -d:)"
echo "::error file=$file,line=$lineno::Copyright out of date"
errors=$((errors + 1))
fi
continue
fi

# User domain copyright does not exist so add it after an existing copyright
did_add_copyright=false
for _regex in "${all_regex_except_user[@]}"; do
read -r y1_other y2_other <<< "$(sed -nre "s/^.*$_regex.*$/\4 \6/p" "$file")"
if [[ -z $y1_other ]] ; then
continue
fi

if [[ "$mode" == "githook" ]]; then
# Add copyright in place, mimicking the format of existing copyright
git_reset "$file" || continue
if [[ -z "$y2_other" ]]; then
y1_y2_other="$y1_other"
else
y1_y2_other="$y1_other-$y2_other"
fi
if [[ "$os" == 'Linux' ]]; then
sed -i -re "s/$_regex/\1Copyright $y1_y2_other \8\n\1Copyright $year $shortname_user/" "$file"
else
sed -i '' -re "s/$_regex/\1Copyright $y1_y2_other \8\n\1Copyright $year $shortname_user/" "$file"
fi
git_add "$file" || continue
elif [[ "$mode" == "gha" ]]; then
# Print error but do not add
lineno="$(grep -nE "$_regex" "$file" | cut -f1 -d:)"
echo "::error file=$file,line=$lineno::Copyright out of date"
errors=$((errors + 1))
fi

did_add_copyright=true
break
done

if ! $did_add_copyright; then
# Print warning but don't error on non-existent copyright since it's not easy to
# determine the format and where to put it
echo " Copyright Information not found in: $file"
fi
done

if [[ $errors -ne 0 ]]; then
echo " $errors errors while checking/fixing copyrights."
exit 1
fi
Loading

0 comments on commit 09dea61

Please sign in to comment.