Make E2E tests more helpful for PRs

PRs are more likely to have genuine failures than commits to master,
so retry them less often and try all the tests even if some fail

.github/workflows/e2e.yml

@@ -4,9 +4,6 @@ on:
   push:
     branches:
       - 'master'
-  pull_request:
-    branches:
-      - '*'
   schedule:
     - cron: "45 7 * * *"
 

.github/workflows/e2e_pr.yml

@@ -0,0 +1,94 @@
+name: e2e_pr
+
+# This is just a duplicate of e2e.yml with fewer tries & no fail-fast so people can get quicker + better feedback on PRs
+# I'm sure there's a neater way of doing it with ifs but this was quicker
+
+on:
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+  test:
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu, windows]
+        # os: [ubuntu, macos, windows]
+        browser: [firefox, firefoxesr]
+        exclude:
+          - os: macos
+            browser: firefoxesr
+
+    runs-on: ${{ matrix.os }}-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@master
+
+    - name: Get yarn cache directory path
+      id: yarn-cache-dir-path
+      run: echo "::set-output name=dir::$(yarn cache dir)"
+
+    - uses: actions/cache@v2
+      id: yarn-cache
+      with:
+        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-yarn-
+
+    - name: Install deps
+      run: |
+        yarn install
+    - name: Install Firefox Dev Edition (Linux)
+      if: matrix.browser == 'firefox' && matrix.os == 'ubuntu'
+      run: |
+        sudo yarn global add get-firefox
+        get-firefox --platform linux --branch devedition --extract --target $HOME
+        echo "$HOME/firefox" >> $GITHUB_PATH
+    - name: Install Firefox ESR (Linux)
+      if: matrix.browser == 'firefoxesr' && matrix.os == 'ubuntu'
+      run: |
+        sudo yarn global add get-firefox
+        get-firefox --platform linux --branch esr --extract --target $HOME
+        echo "$HOME/firefox" >> $GITHUB_PATH
+    - name: Install Firefox Dev Edition (MacOS)
+      if: matrix.browser == 'firefox' && matrix.os == 'macos'
+      run: |
+        brew install --cask homebrew/cask-versions/firefox-developer-edition
+        echo "/Applications/Firefox Developer Edition.app/Contents/MacOS/" >> $GITHUB_PATH
+    - name: Install Firefox Dev Edition (Windows)
+      if: matrix.browser == 'firefox' && matrix.os == 'windows'
+      run: |
+        choco install firefox-dev --pre
+        echo "C:\Program Files\Firefox Dev Edition" >> $GITHUB_PATH
+    - name: Install Firefox ESR (Windows)
+      if: matrix.browser == 'firefoxesr' && matrix.os == 'windows'
+      run: |
+        choco install firefoxesr
+        echo "C:\Program Files\Mozilla Firefox" >> $GITHUB_PATH
+
+    - name: Print Firefox version (Unix-like)
+      if: matrix.os == 'ubuntu' || matrix.os == 'macos'
+      run: firefox --version
+
+    - name: Build and test (Firefox)
+      uses: nick-invision/retry@v1
+      if: matrix.browser != 'chrome'
+      env:
+        HEADLESS: 1
+      with: 
+        max_attempts: 1
+        timeout_minutes: 10
+        retry_wait_seconds: 10
+        # clearing setupFilesAfterEnv removes the fail-fast script
+        command: |
+          yarn run clean && yarn run build --old-native && yarn make-zip && yarn jest --setupFilesAfterEnv=""
+    # - name: Test (Chrome, Linux)
+    #   if: matrix.browser == 'chrome' && matrix.os == 'ubuntu'
+    #   run: xvfb-run --auto-servernum npm run jest -- ${{ matrix.browser }} || xvfb-run --auto-servernum npm run jest -- ${{ matrix.browser }}
+    # - name: Test (Chrome, Windows/MacOs)
+    #   if: matrix.browser == 'chrome' && matrix.os != 'ubuntu'
+    #   run: npm run jest -- ${{ matrix.browser }} || npm run jest -- ${{ matrix.browser }}

e2e_tests/e2e.test.ts

@@ -33,6 +33,7 @@ describe("webdriver", () => {
         const options = (new Options())
                 .setPreference("xpinstall.signatures.required", false)
                 .addExtensions(extensionPath)
+        // options.setBinary("/usr/bin/firefox-developer-edition") // set this if running locally :)
         if (env["HEADLESS"]) {
             options.headless();
         }