Re: [PATCH] s3-mirror: Run crons to sync s3 mirror a lot more often

We have been getting complaints from copr users that they are hitting out of date cloudfront cached data when they are doing builds. We are syncing not all that often currently, and sometimes if a updates push or rawhide compose finishes after the sync time it coud be a long while before it picks up on it. So, since most of these jobs finish in 5-10min when there is nothing to sync, just have them all run every 15min or so. If this starts hitting locking too much we can spread them back out once we get a sense of when they are hitting that. Additionally, we should just set them up to only sync when their particular thing has finished. This would make it a lot more sane, but require a redesign/rewrite. Amended to use the old values for test releases. Signed-off-by: Kevin Fenzi <kevin(a)scrye.com&gt; --- roles/s3-mirror/tasks/main.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/roles/s3-mirror/tasks/main.yml b/roles/s3-mirror/tasks/main.yml index 12351cb..075785e 100644 --- a/roles/s3-mirror/tasks/main.yml +++ b/roles/s3-mirror/tasks/main.yml @@ -68,7 +68,7 @@ - s3-mirror - name: s3sync cron - updates for current - cron: name="s3sync-updates-current" minute="0" hour="3,9,15,21" user="s3-mirror" + cron: name="s3sync-updates-current" minute="2,17,32,47" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-updates-current "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int }}/" 2>&1 | /usr/local/bin/nag-once s3-updates-current.sh 1d 2>&1' cron_file=s3-updates-current.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.') @@ -76,7 +76,7 @@ - s3-mirror - name: s3sync cron - updates for development/current+1 x86_64 - cron: name="s3sync-updates-current" minute="0" hour="2,7,10" user="s3-mirror" + cron: name="s3sync-updates-current" minute="3,18,33,48" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-updates-dev-cur-plus-1-x86_64 "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/development/{{ FedoraCycleNumber|int + 1 }}/Everything/x86_64/os/" 2>&1 | /usr/local/bin/nag-once s3-updates-dev-cur-plus-1-x86_64.sh 1d 2>&1' cron_file=s3-updates-dev-cur-plus-1-x86_64.sh disabled={{not FedoraBranched|bool}} @@ -85,7 +85,7 @@ - s3-mirror - name: s3sync cron - updates for development/current+1 aarch64 - cron: name="s3sync-updates-current" minute="0" hour="4,11,18" user="s3-mirror" + cron: name="s3sync-updates-current" minute="4,19,34,49" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-updates-dev-cur-plus-1-aarch64 "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/development/{{ FedoraCycleNumber|int + 1 }}/Everything/aarch64/os/" 2>&1 | /usr/local/bin/nag-once s3-updates-dev-cur-plus-1-aarch64.sh 1d 2>&1' cron_file=s3-updates-dev-cur-plus-1-aarch64.sh disabled={{not FedoraBranched|bool}} @@ -94,7 +94,7 @@ - s3-mirror - name: s3sync cron - updates for current-1 - cron: name="s3sync-updates-previous" minute="30" hour="0,6,12,18" user="s3-mirror" + cron: name="s3sync-updates-previous" minute="5,20,35,50" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-updates-previous "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int - 1 }}/" 2>&1 | /usr/local/bin/nag-once s3-updates-previous.sh 1d 2>&1' cron_file=s3-updates-previous.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.') @@ -102,7 +102,7 @@ - s3-mirror - name: s3sync cron - epel 7 x86_64 - cron: name="s3sync-epel7-x86_64" minute="10" hour="2,5,8,11,14,17,20,23" user="s3-mirror" + cron: name="s3sync-epel7-x86_64" minute="6,21,36,51" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-epel7-x86_64 "/usr/local/bin/s3-sync-path.sh /pub/epel/7/x86_64/" 2>&1 | /usr/local/bin/nag-once s3-epel7-x86_64.sh 1d 2>&1' cron_file=s3-epel7-x86_64.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.') @@ -110,7 +110,7 @@ - s3-mirror - name: s3sync cron - epel 7 aarch64 - cron: name="s3sync-epel7-aarch64" minute="20" hour="4,7,10,13,16,19,22" user="s3-mirror" + cron: name="s3sync-epel7-aarch64" minute="7,22,37,52" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-epel7-aarch64 "/usr/local/bin/s3-sync-path.sh /pub/epel/7/aarch64/" 2>&1 | /usr/local/bin/nag-once s3-epel7-aarch64.sh 1d 2>&1' cron_file=s3-epel7-aarch64.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.') @@ -118,7 +118,7 @@ - s3-mirror - name: s3sync cron - epel 8 Everything x86_64 - cron: name="s3sync-epel8-everything-x86_64" minute="43" hour="3,6,9,12,15,17,20,23" user="s3-mirror" + cron: name="s3sync-epel8-everything-x86_64" minute="8,23,38,53" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-epel8-everything-x86_64 "/usr/local/bin/s3-sync-path.sh /pub/epel/8/Everything/x86_64/" 2>&1 | /usr/local/bin/nag-once s3-epel8-everything-x86_64.sh 1d 2>&1' cron_file=s3-epel8-everything-x86_64.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.') @@ -126,7 +126,7 @@ - s3-mirror - name: s3sync cron - epel 8 Everything aarch64 - cron: name="s3sync-epel8-everything-aarch64" minute="38" hour="4,7,10,13,16,19,22" user="s3-mirror" + cron: name="s3sync-epel8-everything-aarch64" minute="9,24,39,54" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-epel8-everything-aarch64 "/usr/local/bin/s3-sync-path.sh /pub/epel/8/Everything/aarch64/" 2>&1 | /usr/local/bin/nag-once s3-epel8-everything-aarch64.sh 1d 2>&1' cron_file=s3-epel8-everything-aarch64.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.') @@ -134,7 +134,7 @@ - s3-mirror - name: s3sync cron - epel 8 Modular x86_64 - cron: name="s3sync-epel8-modular-x86_64" minute="32" hour="3,6,9,12,15,17,20,23" user="s3-mirror" + cron: name="s3sync-epel8-modular-x86_64" minute="10,25,40,55" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-epel8-modular-x86_64 "/usr/local/bin/s3-sync-path.sh /pub/epel/8/Modular/x86_64/" 2>&1 | /usr/local/bin/nag-once s3-epel8-modular-x86_64.sh 1d 2>&1' cron_file=s3-epel8-modular-x86_64.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.') @@ -142,7 +142,7 @@ - s3-mirror - name: s3sync cron - epel 8 Modular aarch64 - cron: name="s3sync-epel8-modular-aarch64" minute="27" hour="4,7,10,13,16,19,22" user="s3-mirror" + cron: name="s3sync-epel8-modular-aarch64" minute="11,26,41,56" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-epel8-modular-aarch64 "/usr/local/bin/s3-sync-path.sh /pub/epel/8/Modular/aarch64/" 2>&1 | /usr/local/bin/nag-once s3-epel8-modular-aarch64.sh 1d 2>&1' cron_file=s3-epel8-modular-aarch64.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.') -- 1.8.3.1 _______________________________________________ infrastructure mailing list -- infrastructure(a)lists.fedoraproject.org To unsubscribe send an email to infrastructure-leave(a)lists.fedoraproject.org Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/ List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines List Archives: https://lists.fedoraproject.org/archives/list/infrastructure@lists.fedora...