diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 13b16716..955653db 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,14 +9,18 @@ on: pull_request: branches: [ main ] +env: + MAX_THRESHOLD_SIZE: 10240 + jobs: build: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: python-version: [3.6, 3.7, 3.8, 3.9] + os: [ ubuntu-latest, macOS-latest, windows-latest ] steps: - uses: actions/checkout@v2 @@ -25,12 +29,14 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies + shell: bash run: | python -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi pip install 'dvc' - name: Lint with flake8 + shell: bash run: | # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics @@ -41,11 +47,17 @@ jobs: count=$(echo "$OUTPUT" | tail -1) # Fail test explicitly if count >= 10 if [[ $count -ge 10 ]]; then exit $count; fi + - name: windows env variables + if: startsWith(matrix.os, 'windows') + run: | + set TMP=%USERPROFILE%\AppData\Local\Temp + set TEMP=%USERPROFILE%\AppData\Local\Temp - name: Setup env for tests + shell: bash run: | git config --global user.email "you@example.com" git config --global user.name "Your Name" - name: Test with pytest + shell: bash run: | - export MAX_THRESHOLD_SIZE=10240 pytest diff --git a/fds/services/dvc_service.py b/fds/services/dvc_service.py index a6fa7bdd..54a2e976 100644 --- a/fds/services/dvc_service.py +++ b/fds/services/dvc_service.py @@ -79,7 +79,7 @@ def __should_skip_list_add(self, directory: str) -> bool: :param directory: the name of the dir :return: True if we should skip, else return False """ - if os.path.abspath(directory) == self.repo_path: + if os.path.samefile(os.path.abspath(directory), self.repo_path): return True git_output = check_git_ignore(directory) if convert_bytes_to_string(git_output.stdout) != '': diff --git a/fds/utils.py b/fds/utils.py index c470ef28..4e8beeb5 100644 --- a/fds/utils.py +++ b/fds/utils.py @@ -1,11 +1,12 @@ import getpass import subprocess +import threading from pathlib import Path +import queue import os from typing import List, Union, Any, Optional, Dict import humanize -import select import sys from fds.logger import Logger @@ -27,28 +28,37 @@ def convert_bytes_to_string(bytes_data: bytes) -> str: def execute_command(command: Union[str, List[str]], shell: bool = False, capture_output: bool = True, ignorable_return_codes: List[int] = [0], capture_output_and_write_to_stdout: bool = False) -> Any: + + # Helper functions to read from stdout + def read_output(pipe, funcs): + for line in iter(pipe.readline, ''): + for func in funcs: + func(line) + pipe.close() + # Helper functions to write to stdout + def write_output(get): + for line in iter(get, None): + sys.stdout.write(line) + if capture_output: # capture_output is not available in python 3.6, so using PIPE manually output = subprocess.run(command, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE) elif capture_output_and_write_to_stdout: output = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout = [] - stderr = [] - while True: - reads = [output.stdout.fileno(), output.stderr.fileno()] - ret = select.select(reads, [], []) - - for fd in ret[0]: - if fd == output.stdout.fileno(): - read = output.stdout.readline() - sys.stdout.write(convert_bytes_to_string(read)) - stdout.append(read) - if fd == output.stderr.fileno(): - read = output.stderr.readline() - sys.stderr.write(convert_bytes_to_string(read)) - stderr.append(read) - if output.poll() is not None: - break + q = queue.Queue() + stdout, stderr = [], [] + tout = threading.Thread( + target=read_output, args=(output.stdout, [q.put, stdout.append])) + terr = threading.Thread( + target=read_output, args=(output.stderr, [q.put, stderr.append])) + twrite = threading.Thread(target=write_output, args=(q.get,)) + for t in (tout, terr, twrite): + t.daemon = True + t.start() + output.communicate() + for t in (tout, terr): + t.join() + q.put(None) # create a completed process to have same convention return subprocess.CompletedProcess(command, output.returncode, b''.join(stdout), b''.join(stderr)) else: diff --git a/tests/it/helpers.py b/tests/it/helpers.py index 3d6a7cc4..f95c9aec 100644 --- a/tests/it/helpers.py +++ b/tests/it/helpers.py @@ -26,7 +26,11 @@ def re_init_services(self): def tearDown(self): super().tearDown() - shutil.rmtree(self.repo_path) + try: + shutil.rmtree(self.repo_path) + except Exception as e: + # In windows rmtree doesn't work fine, so using a command directly + os.system('rmdir /S /Q "{}"'.format(self.repo_path)) def create_fake_git_data(self): git_path = f"{self.repo_path}/git_data" diff --git a/tests/it/test_dvc.py b/tests/it/test_dvc.py index 9d0d0299..c71a1232 100644 --- a/tests/it/test_dvc.py +++ b/tests/it/test_dvc.py @@ -176,7 +176,7 @@ def test_get_repo_path(self): self.git_service.init() self.dvc_service.init() path = self.dvc_service.get_repo_path() - assert path == self.repo_path + assert os.path.samefile(self.repo_path, path) self.create_dummy_folder("test_dvc") path = self.dvc_service.get_repo_path() - assert path == self.repo_path + assert os.path.samefile(self.repo_path, path) diff --git a/tests/it/test_git.py b/tests/it/test_git.py index 4fb037b0..9ebc56bb 100644 --- a/tests/it/test_git.py +++ b/tests/it/test_git.py @@ -1,3 +1,5 @@ +import os + from fds.utils import does_file_exist, execute_command, convert_bytes_to_string from tests.it.helpers import IntegrationTestCase @@ -97,7 +99,7 @@ def test_clone_with_dir(self): def test_get_repo_path(self): self.git_service.init() path = self.git_service.get_repo_path() - assert path == self.repo_path + assert os.path.samefile(self.repo_path, path) self.create_dummy_folder("test_git") path = self.git_service.get_repo_path() - assert path == self.repo_path + assert os.path.samefile(self.repo_path, path)