From 46c8f9fe842f1ad822fdd13455c68f5627a314b3 Mon Sep 17 00:00:00 2001 From: Luke Ceddia Date: Mon, 1 Feb 2021 20:47:45 +1100 Subject: [PATCH 1/8] Add aws action --- .ci/deploy.sh | 17 ++++++----------- .github/workflows/run.yml | 20 ++++++++------------ 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/.ci/deploy.sh b/.ci/deploy.sh index 9cfd56d57..50fc2d8b6 100755 --- a/.ci/deploy.sh +++ b/.ci/deploy.sh @@ -2,18 +2,13 @@ OS=$1 TZ=UTC now=`date +"%F-%H-%M-%S"` -filename="/tmp/qb64_${now}_`echo ${GITHUB_SHA} | sed 's/\(.......\).*$/\1/'`_$OS.tar.gz" +BUCKET=qb64-dev-builds +filename="/tmp/qb64_${now}_`echo ${GITHUB_SHA} | sed 's/\(.......\).*$/\1/'`_${OS}.tar.gz" cd .. tar --create --auto-compress --file ${filename} --exclude-from=qb64/.ci/common-exclusion.list --exclude-from=qb64/.ci/$OS-exclusion.list qb64 -# Send to server -# Sometimes the connection can be a bit flakey, so try multiple times on error -for i in `seq 1 10` -do scp ${filename} remote-server:autobuilds/development/ - if [ "$?" -eq 0 ] - then exit 0 - fi - echo scp $i failed +current_files=$(aws --output text --query 'Contents[].Key' s3api list-objects --bucket ${BUCKET} --prefix ${OS}) +aws s3 cp ${filename} s3://${BUCKET}/${OS}/ +for f in current_files; do + aws s3 rm s3://${BUCKET}/$f done -exit 1 - diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 3e345811d..c5906c366 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -7,7 +7,7 @@ on: jobs: linux-build: runs-on: ubuntu-latest - if: "contains(github.event.head_commit.message, 'ci-skip')" + if: "!contains(github.event.head_commit.message, 'ci-skip')" steps: - uses: actions/checkout@v2 - name: Install dependencies @@ -25,18 +25,14 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} branch: development - - name: Set deployment SSH key + - name: Setup aws if: github.event_name == 'push' - uses: shimataro/ssh-key-action@v2.1.0 + uses: aws-actions/configure-aws-credentials@v1 with: - key: ${{ secrets.SSH_KEY }} - # "qb64.org ssh-rsa AAAA" etc. - known_hosts: ${{ secrets.KNOWN_HOSTS }} - config: | - Host remote-server - HostName qb64.org - User m6rosupy1q2t - - name: Package and deploy + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Publish to S3 if: github.event_name == 'push' run: .ci/deploy.sh lnx @@ -90,7 +86,7 @@ jobs: windows-x64-build: runs-on: windows-latest - if: "!contains(github.event.head_commit.message, 'ci-skip')" + if: "contains(github.event.head_commit.message, 'ci-skip')" env: PLATFORM: x64 steps: From 93645ed7e6aa30ea75cc9191219594337789e536 Mon Sep 17 00:00:00 2001 From: Luke Ceddia Date: Mon, 1 Feb 2021 20:58:41 +1100 Subject: [PATCH 2/8] Add osx --- .ci/deploy.sh | 2 +- .github/workflows/run.yml | 17 +++++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.ci/deploy.sh b/.ci/deploy.sh index 50fc2d8b6..f8b1c77cd 100755 --- a/.ci/deploy.sh +++ b/.ci/deploy.sh @@ -9,6 +9,6 @@ tar --create --auto-compress --file ${filename} --exclude-from=qb64/.ci/common-e current_files=$(aws --output text --query 'Contents[].Key' s3api list-objects --bucket ${BUCKET} --prefix ${OS}) aws s3 cp ${filename} s3://${BUCKET}/${OS}/ -for f in current_files; do +for f in $current_files; do aws s3 rm s3://${BUCKET}/$f done diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index c5906c366..6d36c81e1 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -38,24 +38,21 @@ jobs: macos-build: runs-on: macos-latest - if: "contains(github.event.head_commit.message, 'ci-skip')" + if: "!contains(github.event.head_commit.message, 'ci-skip')" steps: - uses: actions/checkout@v2 - name: Bootstrap compiler run: .ci/bootstrap.sh osx - name: Compile run: .ci/compile.sh - - name: Set deployment SSH key + - name: Setup aws if: github.event_name == 'push' - uses: shimataro/ssh-key-action@v2.1.0 + uses: aws-actions/configure-aws-credentials@v1 with: - key: ${{ secrets.SSH_KEY }} - known_hosts: ${{ secrets.KNOWN_HOSTS }} - config: | - Host remote-server - HostName qb64.org - User m6rosupy1q2t - - name: Package and deploy + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Publish to S3 if: github.event_name == 'push' run: .ci/deploy.sh osx From 3ed73b2cfec33ce63183489a37f6e856f8eb1b3e Mon Sep 17 00:00:00 2001 From: Luke Ceddia Date: Mon, 1 Feb 2021 21:11:02 +1100 Subject: [PATCH 3/8] Add windows x86 --- .ci/deploy.bat | 24 ++---------------------- .github/workflows/run.yml | 17 +++++++---------- 2 files changed, 9 insertions(+), 32 deletions(-) diff --git a/.ci/deploy.bat b/.ci/deploy.bat index 3461a9e07..0466dd830 100644 --- a/.ci/deploy.bat +++ b/.ci/deploy.bat @@ -1,29 +1,9 @@ @ECHO OFF SETLOCAL ENABLEEXTENSIONS ENABLEDELAYEDEXPANSION cd .. +set BUCKET=qb64-dev-builds set TIMEX=%TIME: =0% set archive=qb64_%DATE:~10,4%-%DATE:~4,2%-%DATE:~7,2%-%TIMEX:~0,2%-%TIMEX:~3,2%-%TIMEX:~6,2%_%GITHUB_SHA:~0,7%_win-%PLATFORM%.7z 7z a -xr@qb64\.ci\common-exclusion.list -xr@qb64\.ci\win-exclusion.list %archive% qb64 -FOR /L %%i IN (1,1,10) DO ( - scp %archive% remote-server:autobuilds/development/ - IF ERRORLEVEL 1 ( - ECHO scp %archive% failed - attempt %%i/10 - ) ELSE ( - GOTO :doversion - ) -) - -:doversion - -set archive=versioninfo.txt -echo Dev build generated on %DATE:~10,4%-%DATE:~4,2%-%DATE:~7,2%, from git ^%GITHUB_SHA:~0,7%^ > %archive% - -FOR /L %%i IN (1,1,10) DO ( - scp %archive% remote-server:autobuilds/development/ - IF ERRORLEVEL 1 ( - ECHO scp %archive% failed - attempt %%i/10 - ) ELSE ( - EXIT /B 0 - ) -) +aws s3 cp %archive% s3://%BUCKET%/win-%PLATFORM%/ diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 6d36c81e1..0cd780ed8 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -58,7 +58,7 @@ jobs: windows-x86-build: runs-on: windows-latest - if: "contains(github.event.head_commit.message, 'ci-skip')" + if: "!contains(github.event.head_commit.message, 'ci-skip')" env: PLATFORM: x86 steps: @@ -67,17 +67,14 @@ jobs: run: .ci/bootstrap.bat - name: Compile run: .ci/compile.bat - - name: Set deployment SSH key + - name: Setup aws if: github.event_name == 'push' - uses: shimataro/ssh-key-action@v2.1.0 + uses: aws-actions/configure-aws-credentials@v1 with: - key: ${{ secrets.SSH_KEY }} - known_hosts: ${{ secrets.KNOWN_HOSTS }} - config: | - Host remote-server - HostName qb64.org - User m6rosupy1q2t - - name: Package and deploy + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Publish to S3 if: github.event_name == 'push' run: .ci/deploy.bat From eb5d78194bfb250cfbcbc40faa98e6c6226ddc95 Mon Sep 17 00:00:00 2001 From: Luke Ceddia Date: Tue, 2 Feb 2021 21:02:55 +1100 Subject: [PATCH 4/8] Add win file deletion --- .ci/deploy.bat | 4 ++++ .github/workflows/run.yml | 42 +++++++++++---------------------------- 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/.ci/deploy.bat b/.ci/deploy.bat index 0466dd830..1e4b05d77 100644 --- a/.ci/deploy.bat +++ b/.ci/deploy.bat @@ -6,4 +6,8 @@ set TIMEX=%TIME: =0% set archive=qb64_%DATE:~10,4%-%DATE:~4,2%-%DATE:~7,2%-%TIMEX:~0,2%-%TIMEX:~3,2%-%TIMEX:~6,2%_%GITHUB_SHA:~0,7%_win-%PLATFORM%.7z 7z a -xr@qb64\.ci\common-exclusion.list -xr@qb64\.ci\win-exclusion.list %archive% qb64 +aws --output text --query 'Contents[].Key' s3api list-objects --bucket %BUCKET% --prefix win-%PLATFORM% > olddevbuilds.txt + aws s3 cp %archive% s3://%BUCKET%/win-%PLATFORM%/ + +for /F "tokens=*" %%G in (olddevbuilds.txt) do aws s3 rm s3://%BUCKET%/%%G diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 0cd780ed8..0738d73c4 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -7,7 +7,7 @@ on: jobs: linux-build: runs-on: ubuntu-latest - if: "!contains(github.event.head_commit.message, 'ci-skip')" + if: "contains(github.event.head_commit.message, 'ci-skip')" steps: - uses: actions/checkout@v2 - name: Install dependencies @@ -38,7 +38,7 @@ jobs: macos-build: runs-on: macos-latest - if: "!contains(github.event.head_commit.message, 'ci-skip')" + if: "contains(github.event.head_commit.message, 'ci-skip')" steps: - uses: actions/checkout@v2 - name: Bootstrap compiler @@ -85,35 +85,17 @@ jobs: PLATFORM: x64 steps: - uses: actions/checkout@v2 - #- name: Bootstrap compiler - #run: .ci/bootstrap.bat - #- name: Compile - #run: .ci/compile.bat - - name: Set deployment SSH key + - name: Bootstrap compiler + run: .ci/bootstrap.bat + - name: Compile + run: .ci/compile.bat + - name: Setup aws if: github.event_name == 'push' - uses: shimataro/ssh-key-action@v2.1.0 + uses: aws-actions/configure-aws-credentials@v1 with: - key: ${{ secrets.SSH_KEY }} - known_hosts: ${{ secrets.KNOWN_HOSTS }} - config: | - Host remote-server - HostName qb64.org - User m6rosupy1q2t - - name: Download - run: Invoke-WebRequest https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-windows-amd64.zip -OutFile ngrok.zip - - name: Extract - run: Expand-Archive ngrok.zip - - name: Auth - run: .\ngrok\ngrok.exe authtoken $Env:NGROK_AUTH_TOKEN - env: - NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }} - - name: Enable TS - run: Set-ItemProperty -Path 'HKLM:\System\CurrentControlSet\Control\Terminal Server'-name "fDenyTSConnections" -Value 0 - - run: Enable-NetFirewallRule -DisplayGroup "Remote Desktop" - - run: Set-ItemProperty -Path 'HKLM:\System\CurrentControlSet\Control\Terminal Server\WinStations\RDP-Tcp' -name "UserAuthentication" -Value 1 - - run: Set-LocalUser -Name "runneradmin" -Password (ConvertTo-SecureString -AsPlainText "P@ssw0rd!" -Force) - - name: Create Tunnel - run: .\ngrok\ngrok.exe tcp 3389 - - name: Package and deploy + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Publish to S3 if: github.event_name == 'push' run: .ci/deploy.bat From 36a0a28e5926b64b78b3a1e61159ab2bbf49737e Mon Sep 17 00:00:00 2001 From: Luke Ceddia Date: Tue, 2 Feb 2021 21:36:46 +1100 Subject: [PATCH 5/8] Remove quotes? --- .ci/deploy.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/deploy.bat b/.ci/deploy.bat index 1e4b05d77..4d3b2559c 100644 --- a/.ci/deploy.bat +++ b/.ci/deploy.bat @@ -6,7 +6,7 @@ set TIMEX=%TIME: =0% set archive=qb64_%DATE:~10,4%-%DATE:~4,2%-%DATE:~7,2%-%TIMEX:~0,2%-%TIMEX:~3,2%-%TIMEX:~6,2%_%GITHUB_SHA:~0,7%_win-%PLATFORM%.7z 7z a -xr@qb64\.ci\common-exclusion.list -xr@qb64\.ci\win-exclusion.list %archive% qb64 -aws --output text --query 'Contents[].Key' s3api list-objects --bucket %BUCKET% --prefix win-%PLATFORM% > olddevbuilds.txt +aws --output text --query Contents[].Key s3api list-objects --bucket %BUCKET% --prefix win-%PLATFORM% > olddevbuilds.txt aws s3 cp %archive% s3://%BUCKET%/win-%PLATFORM%/ From e7f4eeffad3dfcb2be8e42ff9a85423657808eea Mon Sep 17 00:00:00 2001 From: Luke Ceddia Date: Tue, 2 Feb 2021 22:39:11 +1100 Subject: [PATCH 6/8] Try some powershell --- .ci/deploy.bat | 13 ------------- .ci/deploy.ps1 | 11 +++++++++++ .github/workflows/run.yml | 2 +- 3 files changed, 12 insertions(+), 14 deletions(-) delete mode 100644 .ci/deploy.bat create mode 100644 .ci/deploy.ps1 diff --git a/.ci/deploy.bat b/.ci/deploy.bat deleted file mode 100644 index 4d3b2559c..000000000 --- a/.ci/deploy.bat +++ /dev/null @@ -1,13 +0,0 @@ -@ECHO OFF -SETLOCAL ENABLEEXTENSIONS ENABLEDELAYEDEXPANSION -cd .. -set BUCKET=qb64-dev-builds -set TIMEX=%TIME: =0% -set archive=qb64_%DATE:~10,4%-%DATE:~4,2%-%DATE:~7,2%-%TIMEX:~0,2%-%TIMEX:~3,2%-%TIMEX:~6,2%_%GITHUB_SHA:~0,7%_win-%PLATFORM%.7z -7z a -xr@qb64\.ci\common-exclusion.list -xr@qb64\.ci\win-exclusion.list %archive% qb64 - -aws --output text --query Contents[].Key s3api list-objects --bucket %BUCKET% --prefix win-%PLATFORM% > olddevbuilds.txt - -aws s3 cp %archive% s3://%BUCKET%/win-%PLATFORM%/ - -for /F "tokens=*" %%G in (olddevbuilds.txt) do aws s3 rm s3://%BUCKET%/%%G diff --git a/.ci/deploy.ps1 b/.ci/deploy.ps1 new file mode 100644 index 000000000..963dd9d77 --- /dev/null +++ b/.ci/deploy.ps1 @@ -0,0 +1,11 @@ +$Bucket = "qb64-dev-builds" +$Timestamp = (Get-Date).ToUniversalTime().ToString("yyyy-MM-dd-HH-mm-ss") +$Filename = "qb64_${Timestamp}_$($Env:GITHUB_SHA.substring(0,7))_win-${Env:PLATFORM}.7z" + +Set-Location .. +7z a -xr@qb64\.ci\common-exclusion.list -xr@qb64\.ci\win-exclusion.list $Filename qb64 +$OldFiles = aws --output json --query Contents[].Key s3api list-objects --bucket $Bucket --prefix win-$Env:PLATFORM | ConvertFrom-Json +aws s3 cp $Filename "s3://${Bucket}/win-${Env:PLATFORM}/" +foreach ($f in $OldFiles) { + aws s3 rm "s3://$Bucket/$f" +} \ No newline at end of file diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 0738d73c4..b13f425ac 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -76,7 +76,7 @@ jobs: aws-region: us-east-2 - name: Publish to S3 if: github.event_name == 'push' - run: .ci/deploy.bat + run: .ci/deploy.ps1 windows-x64-build: runs-on: windows-latest From 5ef682c7ca2237d40820050aa815a4ef72782710 Mon Sep 17 00:00:00 2001 From: Luke Ceddia Date: Tue, 2 Feb 2021 22:52:33 +1100 Subject: [PATCH 7/8] Quote 7z command --- .ci/deploy.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/deploy.ps1 b/.ci/deploy.ps1 index 963dd9d77..2c447911d 100644 --- a/.ci/deploy.ps1 +++ b/.ci/deploy.ps1 @@ -3,7 +3,7 @@ $Timestamp = (Get-Date).ToUniversalTime().ToString("yyyy-MM-dd-HH-mm-ss") $Filename = "qb64_${Timestamp}_$($Env:GITHUB_SHA.substring(0,7))_win-${Env:PLATFORM}.7z" Set-Location .. -7z a -xr@qb64\.ci\common-exclusion.list -xr@qb64\.ci\win-exclusion.list $Filename qb64 +7z a '-xr@qb64\.ci\common-exclusion.list' '-xr@qb64\.ci\win-exclusion.list' $Filename qb64 $OldFiles = aws --output json --query Contents[].Key s3api list-objects --bucket $Bucket --prefix win-$Env:PLATFORM | ConvertFrom-Json aws s3 cp $Filename "s3://${Bucket}/win-${Env:PLATFORM}/" foreach ($f in $OldFiles) { From 39782d347527c3f71b15ab4fa2e83830a035a352 Mon Sep 17 00:00:00 2001 From: Luke Ceddia Date: Tue, 2 Feb 2021 23:00:43 +1100 Subject: [PATCH 8/8] Re-enable other platforms --- .github/workflows/run.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index b13f425ac..992417df8 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -2,12 +2,11 @@ on: push: branches: - development - - ci-test jobs: linux-build: runs-on: ubuntu-latest - if: "contains(github.event.head_commit.message, 'ci-skip')" + if: "!contains(github.event.head_commit.message, 'ci-skip')" steps: - uses: actions/checkout@v2 - name: Install dependencies @@ -38,7 +37,7 @@ jobs: macos-build: runs-on: macos-latest - if: "contains(github.event.head_commit.message, 'ci-skip')" + if: "!contains(github.event.head_commit.message, 'ci-skip')" steps: - uses: actions/checkout@v2 - name: Bootstrap compiler @@ -80,7 +79,7 @@ jobs: windows-x64-build: runs-on: windows-latest - if: "contains(github.event.head_commit.message, 'ci-skip')" + if: "!contains(github.event.head_commit.message, 'ci-skip')" env: PLATFORM: x64 steps: @@ -98,4 +97,4 @@ jobs: aws-region: us-east-2 - name: Publish to S3 if: github.event_name == 'push' - run: .ci/deploy.bat + run: .ci/deploy.ps1