Run tests on windows and mac

.github/workflows/test.yml

@@ -9,14 +9,18 @@ on:
   pull_request:
     branches: [ main ]
 
+env:
+  MAX_THRESHOLD_SIZE: 10240
+
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9]
+        os: [ ubuntu-latest, macOS-latest, windows-latest ]
 
     steps:
     - uses: actions/checkout@v2
@@ -25,12 +29,14 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
+      shell: bash
       run: |
         python -m pip install --upgrade pip
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
         pip install 'dvc'
     - name: Lint with flake8
+      shell: bash
       run: |
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
@@ -41,11 +47,17 @@ jobs:
         count=$(echo "$OUTPUT" | tail -1)
         # Fail test explicitly if count >= 10
         if [[ $count -ge 10 ]]; then exit $count; fi
+    - name: windows env variables
+      if: startsWith(matrix.os, 'windows')
+      run: |
+        set TMP=%USERPROFILE%\AppData\Local\Temp
+        set TEMP=%USERPROFILE%\AppData\Local\Temp
     - name: Setup env for tests
+      shell: bash
       run: |
         git config --global user.email "you@example.com"
         git config --global user.name "Your Name"
     - name: Test with pytest
+      shell: bash
       run: |
-        export MAX_THRESHOLD_SIZE=10240
         pytest

fds/services/dvc_service.py

@@ -79,7 +79,7 @@ def __should_skip_list_add(self, directory: str) -> bool:
         :param directory: the name of the dir
         :return: True if we should skip, else return False
         """
-        if os.path.abspath(directory) == self.repo_path:
+        if os.path.samefile(os.path.abspath(directory), self.repo_path):
             return True
         git_output = check_git_ignore(directory)
         if convert_bytes_to_string(git_output.stdout) != '':

fds/utils.py

@@ -1,11 +1,12 @@
 import getpass
 import subprocess
+import threading
 from pathlib import Path
+import queue
 import os
 from typing import List, Union, Any, Optional, Dict
 
 import humanize
-import select
 import sys
 from fds.logger import Logger
 
@@ -27,28 +28,37 @@ def convert_bytes_to_string(bytes_data: bytes) -> str:
 
 def execute_command(command: Union[str, List[str]], shell: bool = False, capture_output: bool = True,
                     ignorable_return_codes: List[int] = [0], capture_output_and_write_to_stdout: bool = False) -> Any:
+
+    # Helper functions to read from stdout
+    def read_output(pipe, funcs):
+        for line in iter(pipe.readline, ''):
+            for func in funcs:
+                func(line)
+        pipe.close()
+    # Helper functions to write to stdout
+    def write_output(get):
+        for line in iter(get, None):
+            sys.stdout.write(line)
+
     if capture_output:
         # capture_output is not available in python 3.6, so using PIPE manually
         output = subprocess.run(command, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     elif capture_output_and_write_to_stdout:
         output = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout = []
-        stderr = []
-        while True:
-            reads = [output.stdout.fileno(), output.stderr.fileno()]
-            ret = select.select(reads, [], [])
-
-            for fd in ret[0]:
-                if fd == output.stdout.fileno():
-                    read = output.stdout.readline()
-                    sys.stdout.write(convert_bytes_to_string(read))
-                    stdout.append(read)
-                if fd == output.stderr.fileno():
-                    read = output.stderr.readline()
-                    sys.stderr.write(convert_bytes_to_string(read))
-                    stderr.append(read)
-            if output.poll() is not None:
-                break
+        q = queue.Queue()
+        stdout, stderr = [], []
+        tout = threading.Thread(
+            target=read_output, args=(output.stdout, [q.put, stdout.append]))
+        terr = threading.Thread(
+            target=read_output, args=(output.stderr, [q.put, stderr.append]))
+        twrite = threading.Thread(target=write_output, args=(q.get,))
+        for t in (tout, terr, twrite):
+            t.daemon = True
+            t.start()
+        output.communicate()
+        for t in (tout, terr):
+            t.join()
+        q.put(None)
         # create a completed process to have same convention
         return subprocess.CompletedProcess(command, output.returncode, b''.join(stdout), b''.join(stderr))
     else:

tests/it/helpers.py

@@ -26,7 +26,11 @@ def re_init_services(self):
 
     def tearDown(self):
         super().tearDown()
-        shutil.rmtree(self.repo_path)
+        try:
+            shutil.rmtree(self.repo_path)
+        except Exception as e:
+            # In windows rmtree doesn't work fine, so using a command directly
+            os.system('rmdir /S /Q "{}"'.format(self.repo_path))
 
     def create_fake_git_data(self):
         git_path = f"{self.repo_path}/git_data"

tests/it/test_dvc.py

@@ -176,7 +176,7 @@ def test_get_repo_path(self):
         self.git_service.init()
         self.dvc_service.init()
         path = self.dvc_service.get_repo_path()
-        assert path == self.repo_path
+        assert os.path.samefile(self.repo_path, path)
         self.create_dummy_folder("test_dvc")
         path = self.dvc_service.get_repo_path()
-        assert path == self.repo_path
+        assert os.path.samefile(self.repo_path, path)

tests/it/test_git.py

@@ -1,3 +1,5 @@
+import os
+
 from fds.utils import does_file_exist, execute_command, convert_bytes_to_string
 from tests.it.helpers import IntegrationTestCase
 
@@ -97,7 +99,7 @@ def test_clone_with_dir(self):
     def test_get_repo_path(self):
         self.git_service.init()
         path = self.git_service.get_repo_path()
-        assert path == self.repo_path
+        assert os.path.samefile(self.repo_path, path)
         self.create_dummy_folder("test_git")
         path = self.git_service.get_repo_path()
-        assert path == self.repo_path
+        assert os.path.samefile(self.repo_path, path)