From b4c7d4131e3a06359e0a97c5d0df4da8f93646f9 Mon Sep 17 00:00:00 2001
From: lihsai0 <lihsai0@gmail.com>
Date: Wed, 9 Oct 2024 15:46:20 +0800
Subject: [PATCH 1/5] fix: windows compatibility
- fix file write error
- fix datetime to timestamp error
- fix read file line by line then json.parse error
- test, add windows test actions
- build, remove useless test packages
- test, fix windows platform not work with unicode characters in codecov config
---
.github/workflows/ci-test.yml | 100 ++++++++++++++++++++++-
codecov.yml | 16 ++--
qiniu/http/regions_provider.py | 11 ++-
qiniu/region.py | 4 +-
qiniu/utils.py | 30 ++++++-
setup.py | 4 -
test_qiniu.py | 123 ----------------------------
tests/cases/test_utils.py | 145 +++++++++++++++++++++++++++++++++
8 files changed, 287 insertions(+), 146 deletions(-)
create mode 100644 tests/cases/test_utils.py
diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
index c705019b..db8752d4 100644
--- a/.github/workflows/ci-test.yml
+++ b/.github/workflows/ci-test.yml
@@ -60,14 +60,106 @@ jobs:
QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}}
QINIU_TEST_ENV: "travis"
MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000"
- PYTHONPATH: "$PYTHONPATH:."
run: |
flake8 --show-source --max-line-length=160 ./qiniu
- coverage run -m pytest ./test_qiniu.py ./tests/cases
- ocular --data-file .coverage
- codecov
+ python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml
+ - name: Post Setup mock server
+ if: ${{ always() }}
+ shell: bash
+ run: |
+ set +e
cat mock-server.pid | xargs kill
+ rm mock-server.pid
- name: Print mock server log
if: ${{ failure() }}
run: |
cat py-mock-server.log
+ - name: Upload results to Codecov
+ uses: codecov/codecov-action@v4
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
+ test-win:
+ strategy:
+ fail-fast: false
+ max-parallel: 1
+ matrix:
+ python_version: ['2.7', '3.5', '3.9']
+ runs-on: windows-2019
+ # make sure only one test running,
+ # remove this when cases could run in parallel.
+ needs: test
+ steps:
+ - name: Checkout repo
+ uses: actions/checkout@v2
+ with:
+ ref: ${{ github.ref }}
+ - name: Setup miniconda
+ uses: conda-incubator/setup-miniconda@v2
+ with:
+ auto-update-conda: true
+ channels: conda-forge
+ python-version: ${{ matrix.python_version }}
+ activate-environment: qiniu-sdk
+ auto-activate-base: false
+ - name: Setup pip
+ env:
+ PYTHON_VERSION: ${{ matrix.python_version }}
+ PIP_BOOTSTRAP_SCRIPT_PREFIX: https://bootstrap.pypa.io/pip
+ run: |
+ # reinstall pip by some python(<3.7) not compatible
+ $pyversion = [Version]"$ENV:PYTHON_VERSION"
+ if ($pyversion -lt [Version]"3.7") {
+ Invoke-WebRequest "$ENV:PIP_BOOTSTRAP_SCRIPT_PREFIX/$($pyversion.Major).$($pyversion.Minor)/get-pip.py" -OutFile "$ENV:TEMP\get-pip.py"
+ python $ENV:TEMP\get-pip.py --user
+ Remove-Item -Path "$ENV:TEMP\get-pip.py"
+ }
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ python -m pip install -I -e ".[dev]"
+ - name: Run cases
+ env:
+ QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }}
+ QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }}
+ QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }}
+ QINIU_TEST_NO_ACC_BUCKET: ${{ secrets.QINIU_TEST_NO_ACC_BUCKET }}
+ QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }}
+ QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}}
+ QINIU_TEST_ENV: "github"
+ MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000"
+ PYTHONPATH: "$PYTHONPATH:."
+ run: |
+ Write-Host "======== Setup Mock Server ========="
+ conda create -y -n mock-server python=3.10
+ conda activate mock-server
+ python --version
+ $processOptions = @{
+ FilePath="python"
+ ArgumentList="tests/mock_server/main.py", "--port", "9000"
+ PassThru=$true
+ RedirectStandardOutput="py-mock-server.log"
+ }
+ $mocksrvp = Start-Process @processOptions
+ $mocksrvp.Id | Out-File -FilePath "mock-server.pid"
+ conda deactivate
+ Write-Host "======== Running Test ========="
+ python --version
+ python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml
+ - name: Post Setup mock server
+ if: ${{ always() }}
+ run: |
+ Try {
+ $mocksrvpid = Get-Content -Path "mock-server.pid"
+ Stop-Process -Id $mocksrvpid
+ Remove-Item -Path "mock-server.pid"
+ } Catch {
+ Write-Host -Object $_
+ }
+ - name: Print mock server log
+ if: ${{ failure() }}
+ run: |
+ Get-Content -Path "py-mock-server.log"
+ - name: Upload results to Codecov
+ uses: codecov/codecov-action@v4
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/codecov.yml b/codecov.yml
index 3f36c50a..0aab28d3 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -1,14 +1,14 @@
codecov:
ci:
- - prow.qiniu.io # prow 里面运行需添加，其他 CI 不要
- require_ci_to_pass: no # 改为 no，否则 codecov 会等待其他 GitHub 上所有 CI 通过才会留言。
+ - prow.qiniu.io # prow need this. seems useless
+ require_ci_to_pass: no # `no` means the bot will comment on the PR even before all ci passed
-github_checks: #关闭github checks
+github_checks: # close github checks
annotations: false
comment:
layout: "reach, diff, flags, files"
- behavior: new # 默认是更新旧留言，改为 new，删除旧的，增加新的。
+ behavior: new # `new` means the bot will comment a new message instead of edit the old one
require_changes: false # if true: only post the comment if coverage changes
require_base: no # [yes :: must have a base report to post]
require_head: yes # [yes :: must have a head report to post]
@@ -16,13 +16,13 @@ comment:
- "master"
coverage:
- status: # 评判 pr 通过的标准
+ status: # check coverage status to pass or fail
patch: off
- project: # project 统计所有代码x
+ project: # project analyze all code in the project
default:
# basic
- target: 73.5% # 总体通过标准
- threshold: 3% # 允许单次下降的幅度
+ target: 73.5% # the minimum coverage ratio that the commit must meet
+ threshold: 3% # allow the coverage to drop
base: auto
if_not_found: success
if_ci_failed: error
diff --git a/qiniu/http/regions_provider.py b/qiniu/http/regions_provider.py
index 8b52822c..13d1800a 100644
--- a/qiniu/http/regions_provider.py
+++ b/qiniu/http/regions_provider.py
@@ -5,9 +5,10 @@
import logging
import tempfile
import os
+import shutil
from qiniu.compat import json, b as to_bytes
-from qiniu.utils import io_md5
+from qiniu.utils import io_md5, dt2ts
from .endpoint import Endpoint
from .region import Region, ServiceName
@@ -264,7 +265,7 @@ def _persist_region(region):
},
ttl=region.ttl,
# use datetime.datetime.timestamp() when min version of python >= 3
- createTime=int(float(region.create_time.strftime('%s.%f')) * 1000)
+ createTime=dt2ts(region.create_time)
)._asdict()
@@ -338,8 +339,10 @@ def _walk_persist_cache_file(persist_path, ignore_parse_error=False):
with open(persist_path, 'r') as f:
for line in f:
+ if not line.strip():
+ continue
try:
- cache_key, regions = _parse_persisted_regions(line)
+ cache_key, regions = _parse_persisted_regions(line.strip())
yield cache_key, regions
except Exception as err:
if not ignore_parse_error:
@@ -655,7 +658,7 @@ def __shrink_cache(self):
)
# rename file
- os.rename(shrink_file_path, self._cache_scope.persist_path)
+ shutil.move(shrink_file_path, self._cache_scope.persist_path)
except FileAlreadyLocked:
pass
finally:
diff --git a/qiniu/region.py b/qiniu/region.py
index 09ac791d..a59d488e 100644
--- a/qiniu/region.py
+++ b/qiniu/region.py
@@ -6,7 +6,7 @@
from .compat import json, s as str_from_bytes
-from .utils import urlsafe_base64_decode
+from .utils import urlsafe_base64_decode, dt2ts
from .config import UC_HOST, is_customized_default, get_default
from .http.endpoint import Endpoint as _HTTPEndpoint
from .http.regions_provider import Region as _HTTPRegion, ServiceName, get_default_regions_provider
@@ -190,7 +190,7 @@ def get_bucket_hosts(self, ak, bucket, home_dir=None, force=False):
ttl = region.ttl if region.ttl > 0 else 24 * 3600 # 1 day
# use datetime.datetime.timestamp() when min version of python >= 3
- create_time = int(float(region.create_time.strftime('%s.%f')) * 1000)
+ create_time = dt2ts(region.create_time)
bucket_hosts['deadline'] = create_time + ttl
return bucket_hosts
diff --git a/qiniu/utils.py b/qiniu/utils.py
index f8517e35..197b8813 100644
--- a/qiniu/utils.py
+++ b/qiniu/utils.py
@@ -1,7 +1,8 @@
# -*- coding: utf-8 -*-
from hashlib import sha1, new as hashlib_new
from base64 import urlsafe_b64encode, urlsafe_b64decode
-from datetime import datetime
+from datetime import datetime, tzinfo, timedelta
+
from .compat import b, s
try:
@@ -236,3 +237,30 @@ def canonical_mime_header_key(field_name):
result += ch
upper = ch == "-"
return result
+
+
+class _UTC_TZINFO(tzinfo):
+ def utcoffset(self, dt):
+ return timedelta(hours=0)
+
+ def tzname(self, dt):
+ return "UTC"
+
+ def dst(self, dt):
+ return timedelta(0)
+
+
+def dt2ts(dt):
+ """
+ converte datetime to timestamp
+
+ Parameters
+ ----------
+ dt: datetime.datetime
+ """
+ if not dt.tzinfo:
+ st = (dt - datetime(1970, 1, 1)).total_seconds()
+ else:
+ st = (dt - datetime(1970, 1, 1, tzinfo=_UTC_TZINFO())).total_seconds()
+
+ return int(st)
diff --git a/setup.py b/setup.py
index cf97eae2..fa920d45 100644
--- a/setup.py
+++ b/setup.py
@@ -42,10 +42,8 @@ def find_version(*file_paths):
'Operating System :: OS Independent',
'Programming Language :: Python',
'Programming Language :: Python :: 2',
- 'Programming Language :: Python :: 2.6',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
- 'Programming Language :: Python :: 3.3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
@@ -66,8 +64,6 @@ def find_version(*file_paths):
'pytest',
'pytest-cov',
'freezegun',
- 'scrutinizer-ocular',
- 'codecov'
]
},
diff --git a/test_qiniu.py b/test_qiniu.py
index 2b71aa22..16f59844 100644
--- a/test_qiniu.py
+++ b/test_qiniu.py
@@ -65,129 +65,6 @@ def remove_temp_file(file):
except OSError:
pass
-
-class UtilsTest(unittest.TestCase):
- def test_urlsafe(self):
- a = 'hello\x96'
- u = urlsafe_base64_encode(a)
- assert b(a) == urlsafe_base64_decode(u)
-
- def test_canonical_mime_header_key(self):
- field_names = [
- ":status",
- ":x-test-1",
- ":x-Test-2",
- "content-type",
- "CONTENT-LENGTH",
- "oRiGin",
- "ReFer",
- "Last-Modified",
- "acCePt-ChArsEt",
- "x-test-3",
- "cache-control",
- ]
- expect_canonical_field_names = [
- ":status",
- ":x-test-1",
- ":x-Test-2",
- "Content-Type",
- "Content-Length",
- "Origin",
- "Refer",
- "Last-Modified",
- "Accept-Charset",
- "X-Test-3",
- "Cache-Control",
- ]
- assert len(field_names) == len(expect_canonical_field_names)
- for i in range(len(field_names)):
- assert canonical_mime_header_key(field_names[i]) == expect_canonical_field_names[i]
-
- def test_entry(self):
- case_list = [
- {
- 'msg': 'normal',
- 'bucket': 'qiniuphotos',
- 'key': 'gogopher.jpg',
- 'expect': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn'
- },
- {
- 'msg': 'key empty',
- 'bucket': 'qiniuphotos',
- 'key': '',
- 'expect': 'cWluaXVwaG90b3M6'
- },
- {
- 'msg': 'key undefined',
- 'bucket': 'qiniuphotos',
- 'key': None,
- 'expect': 'cWluaXVwaG90b3M='
- },
- {
- 'msg': 'key need replace plus symbol',
- 'bucket': 'qiniuphotos',
- 'key': '012ts>a',
- 'expect': 'cWluaXVwaG90b3M6MDEydHM-YQ=='
- },
- {
- 'msg': 'key need replace slash symbol',
- 'bucket': 'qiniuphotos',
- 'key': '012ts?a',
- 'expect': 'cWluaXVwaG90b3M6MDEydHM_YQ=='
- }
- ]
- for c in case_list:
- assert c.get('expect') == entry(c.get('bucket'), c.get('key')), c.get('msg')
-
- def test_decode_entry(self):
- case_list = [
- {
- 'msg': 'normal',
- 'expect': {
- 'bucket': 'qiniuphotos',
- 'key': 'gogopher.jpg'
- },
- 'entry': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn'
- },
- {
- 'msg': 'key empty',
- 'expect': {
- 'bucket': 'qiniuphotos',
- 'key': ''
- },
- 'entry': 'cWluaXVwaG90b3M6'
- },
- {
- 'msg': 'key undefined',
- 'expect': {
- 'bucket': 'qiniuphotos',
- 'key': None
- },
- 'entry': 'cWluaXVwaG90b3M='
- },
- {
- 'msg': 'key need replace plus symbol',
- 'expect': {
- 'bucket': 'qiniuphotos',
- 'key': '012ts>a'
- },
- 'entry': 'cWluaXVwaG90b3M6MDEydHM-YQ=='
- },
- {
- 'msg': 'key need replace slash symbol',
- 'expect': {
- 'bucket': 'qiniuphotos',
- 'key': '012ts?a'
- },
- 'entry': 'cWluaXVwaG90b3M6MDEydHM_YQ=='
- }
- ]
- for c in case_list:
- bucket, key = decode_entry(c.get('entry'))
- assert bucket == c.get('expect').get('bucket'), c.get('msg')
- assert key == c.get('expect').get('key'), c.get('msg')
-
-
class BucketTestCase(unittest.TestCase):
q = Auth(access_key, secret_key)
bucket = BucketManager(q)
diff --git a/tests/cases/test_utils.py b/tests/cases/test_utils.py
new file mode 100644
index 00000000..11d9db77
--- /dev/null
+++ b/tests/cases/test_utils.py
@@ -0,0 +1,145 @@
+from datetime import datetime, timedelta, tzinfo
+
+from qiniu import utils, compat
+
+
+class _CN_TZINFO(tzinfo):
+ def utcoffset(self, dt):
+ return timedelta(hours=8)
+
+ def tzname(self, dt):
+ return "CST"
+
+ def dst(self, dt):
+ return timedelta(0)
+
+
+class TestUtils:
+ def test_urlsafe(self):
+ a = 'hello\x96'
+ u = utils.urlsafe_base64_encode(a)
+ assert compat.b(a) == utils.urlsafe_base64_decode(u)
+
+ def test_canonical_mime_header_key(self):
+ field_names = [
+ ":status",
+ ":x-test-1",
+ ":x-Test-2",
+ "content-type",
+ "CONTENT-LENGTH",
+ "oRiGin",
+ "ReFer",
+ "Last-Modified",
+ "acCePt-ChArsEt",
+ "x-test-3",
+ "cache-control",
+ ]
+ expect_canonical_field_names = [
+ ":status",
+ ":x-test-1",
+ ":x-Test-2",
+ "Content-Type",
+ "Content-Length",
+ "Origin",
+ "Refer",
+ "Last-Modified",
+ "Accept-Charset",
+ "X-Test-3",
+ "Cache-Control",
+ ]
+ assert len(field_names) == len(expect_canonical_field_names)
+ for i in range(len(field_names)):
+ assert utils.canonical_mime_header_key(field_names[i]) == expect_canonical_field_names[i]
+
+ def test_entry(self):
+ case_list = [
+ {
+ 'msg': 'normal',
+ 'bucket': 'qiniuphotos',
+ 'key': 'gogopher.jpg',
+ 'expect': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn'
+ },
+ {
+ 'msg': 'key empty',
+ 'bucket': 'qiniuphotos',
+ 'key': '',
+ 'expect': 'cWluaXVwaG90b3M6'
+ },
+ {
+ 'msg': 'key undefined',
+ 'bucket': 'qiniuphotos',
+ 'key': None,
+ 'expect': 'cWluaXVwaG90b3M='
+ },
+ {
+ 'msg': 'key need replace plus symbol',
+ 'bucket': 'qiniuphotos',
+ 'key': '012ts>a',
+ 'expect': 'cWluaXVwaG90b3M6MDEydHM-YQ=='
+ },
+ {
+ 'msg': 'key need replace slash symbol',
+ 'bucket': 'qiniuphotos',
+ 'key': '012ts?a',
+ 'expect': 'cWluaXVwaG90b3M6MDEydHM_YQ=='
+ }
+ ]
+ for c in case_list:
+ assert c.get('expect') == utils.entry(c.get('bucket'), c.get('key')), c.get('msg')
+
+ def test_decode_entry(self):
+ case_list = [
+ {
+ 'msg': 'normal',
+ 'expect': {
+ 'bucket': 'qiniuphotos',
+ 'key': 'gogopher.jpg'
+ },
+ 'entry': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn'
+ },
+ {
+ 'msg': 'key empty',
+ 'expect': {
+ 'bucket': 'qiniuphotos',
+ 'key': ''
+ },
+ 'entry': 'cWluaXVwaG90b3M6'
+ },
+ {
+ 'msg': 'key undefined',
+ 'expect': {
+ 'bucket': 'qiniuphotos',
+ 'key': None
+ },
+ 'entry': 'cWluaXVwaG90b3M='
+ },
+ {
+ 'msg': 'key need replace plus symbol',
+ 'expect': {
+ 'bucket': 'qiniuphotos',
+ 'key': '012ts>a'
+ },
+ 'entry': 'cWluaXVwaG90b3M6MDEydHM-YQ=='
+ },
+ {
+ 'msg': 'key need replace slash symbol',
+ 'expect': {
+ 'bucket': 'qiniuphotos',
+ 'key': '012ts?a'
+ },
+ 'entry': 'cWluaXVwaG90b3M6MDEydHM_YQ=='
+ }
+ ]
+ for c in case_list:
+ bucket, key = utils.decode_entry(c.get('entry'))
+ assert bucket == c.get('expect', {}).get('bucket'), c.get('msg')
+ assert key == c.get('expect', {}).get('key'), c.get('msg')
+
+ def test_dt2ts(self):
+ dt = datetime(year=2011, month=8, day=3, tzinfo=_CN_TZINFO())
+ expect = 1312300800
+ assert utils.dt2ts(dt) == expect
+
+ base_dt = datetime(year=2011, month=8, day=3)
+ now_dt = datetime.now()
+ assert int((now_dt - base_dt).total_seconds()) == utils.dt2ts(now_dt) - utils.dt2ts(base_dt)
From da2a7bd5ae9745ae2114f30595057bc8a57017e9 Mon Sep 17 00:00:00 2001
From: lihsai0 <lihsai0@gmail.com>
Date: Sat, 12 Oct 2024 11:42:18 +0800
Subject: [PATCH 2/5] test: fix test cases compatibility on Windows
---
.github/workflows/ci-test.yml | 5 +++--
tests/cases/test_http/test_region.py | 2 +-
tests/cases/test_http/test_regions_provider.py | 6 ++++--
3 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
index db8752d4..52195876 100644
--- a/.github/workflows/ci-test.yml
+++ b/.github/workflows/ci-test.yml
@@ -135,13 +135,14 @@ jobs:
python --version
$processOptions = @{
FilePath="python"
- ArgumentList="tests/mock_server/main.py", "--port", "9000"
+ ArgumentList="tests\mock_server\main.py", "--port", "9000"
PassThru=$true
RedirectStandardOutput="py-mock-server.log"
}
$mocksrvp = Start-Process @processOptions
$mocksrvp.Id | Out-File -FilePath "mock-server.pid"
conda deactivate
+ Sleep 3
Write-Host "======== Running Test ========="
python --version
python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml
@@ -158,7 +159,7 @@ jobs:
- name: Print mock server log
if: ${{ failure() }}
run: |
- Get-Content -Path "py-mock-server.log"
+ Get-Content -Path "py-mock-server.log" | Write-Host
- name: Upload results to Codecov
uses: codecov/codecov-action@v4
with:
diff --git a/tests/cases/test_http/test_region.py b/tests/cases/test_http/test_region.py
index a66b16c9..976d2619 100644
--- a/tests/cases/test_http/test_region.py
+++ b/tests/cases/test_http/test_region.py
@@ -36,7 +36,7 @@ def test_custom_options(self):
k in region.services
for k in chain(ServiceName, ['custom-service'])
)
- assert datetime.now() - region.create_time > timedelta(days=1)
+ assert datetime.now() - region.create_time >= timedelta(days=1)
assert region.ttl == 3600
assert not region.is_live
diff --git a/tests/cases/test_http/test_regions_provider.py b/tests/cases/test_http/test_regions_provider.py
index 163f19d2..7289f5ca 100644
--- a/tests/cases/test_http/test_regions_provider.py
+++ b/tests/cases/test_http/test_regions_provider.py
@@ -186,8 +186,10 @@ def test_getter_with_base_regions_provider(self, cached_regions_provider):
assert list(cached_regions_provider) == regions
line_num = 0
with open(cached_regions_provider.persist_path, 'r') as f:
- for _ in f:
- line_num += 1
+ for l in f:
+ # ignore empty line
+ if l.strip():
+ line_num += 1
assert line_num == 1
@pytest.mark.parametrize(
From 2c9c165d19f5bca6e7bda72b50f48f0c6d057d1a Mon Sep 17 00:00:00 2001
From: lihsai0 <lihsai0@gmail.com>
Date: Sat, 12 Oct 2024 14:09:39 +0800
Subject: [PATCH 3/5] test: fix unsetenv not exists in earlier python on
windows
---
test_qiniu.py | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/test_qiniu.py b/test_qiniu.py
index 16f59844..c8dce456 100644
--- a/test_qiniu.py
+++ b/test_qiniu.py
@@ -285,7 +285,11 @@ def test_invalid_x_qiniu_date_with_disable_date_sign(self):
def test_invalid_x_qiniu_date_env(self):
os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = 'True'
ret, info = self.bucket.stat(bucket_name, 'python-sdk.html')
- os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE')
+ if hasattr(os, 'unsetenv'):
+ os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE')
+ else:
+ # fix unsetenv not exists in earlier python on windows
+ os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = ''
assert 'hash' in ret
@freeze_time("1970-01-01")
@@ -294,7 +298,11 @@ def test_invalid_x_qiniu_date_env_be_ignored(self):
q = Auth(access_key, secret_key, disable_qiniu_timestamp_signature=False)
bucket = BucketManager(q)
ret, info = bucket.stat(bucket_name, 'python-sdk.html')
- os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE')
+ if hasattr(os, 'unsetenv'):
+ os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE')
+ else:
+ # fix unsetenv not exists in earlier python on windows
+ os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = ''
assert ret is None
assert info.status_code == 403
From a26c833514345d571b0bddf4efb6c91675441bac Mon Sep 17 00:00:00 2001
From: lihsai0 <lihsai0@gmail.com>
Date: Mon, 14 Oct 2024 20:07:02 +0800
Subject: [PATCH 4/5] feat: add workflow template id for pfop
---
.gitignore | 3 +-
qiniu/auth.py | 3 +-
qiniu/services/processing/pfop.py | 19 ++++-
.../test_processing/test_pfop.py | 76 +++++++++++++++----
.../test_storage/test_upload_pfop.py | 66 +++++++++++-----
5 files changed, 128 insertions(+), 39 deletions(-)
diff --git a/.gitignore b/.gitignore
index 93665221..261e665c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,6 +36,7 @@ pip-log.txt
.coverage
.tox
nosetests.xml
+coverage.xml
# Translations
*.mo
@@ -45,4 +46,4 @@ nosetests.xml
.project
.pydevproject
/.idea
-/.venv
\ No newline at end of file
+/.venv*
diff --git a/qiniu/auth.py b/qiniu/auth.py
index d3e0a055..1647199e 100644
--- a/qiniu/auth.py
+++ b/qiniu/auth.py
@@ -34,10 +34,11 @@
str('fsizeMin'), # 上传文件最少字节数
str('keylimit'), # 设置允许上传的key列表，字符串数组类型，数组长度不可超过20个，如果设置了这个字段，上传时必须提供key
- str('persistentOps'), # 持久化处理操作
+ str('persistentOps'), # 持久化处理操作，与 persistentWorkflowTemplateID 二选一
str('persistentNotifyUrl'), # 持久化处理结果通知URL
str('persistentPipeline'), # 持久化处理独享队列
str('persistentType'), # 为 `1` 时，开启闲时任务，必须是 int 类型
+ str('persistentWorkflowTemplateID'), # 工作流模板 ID，与 persistentOps 二选一
str('deleteAfterDays'), # 文件多少天后自动删除
str('fileType'), # 文件的存储类型，0为标准存储，1为低频存储，2为归档存储，3为深度归档存储，4为归档直读存储
diff --git a/qiniu/services/processing/pfop.py b/qiniu/services/processing/pfop.py
index 346e6277..4b2641e2 100644
--- a/qiniu/services/processing/pfop.py
+++ b/qiniu/services/processing/pfop.py
@@ -24,7 +24,7 @@ def __init__(self, auth, bucket, pipeline=None, notify_url=None):
self.pipeline = pipeline
self.notify_url = notify_url
- def execute(self, key, fops, force=None, persistent_type=None):
+ def execute(self, key, fops=None, force=None, persistent_type=None, workflow_template_id=None):
"""
执行持久化处理
@@ -32,28 +32,39 @@ def execute(self, key, fops, force=None, persistent_type=None):
----------
key: str
待处理的源文件
- fops: list[str]
+ fops: list[str], optional
处理详细操作，规格详见 https://developer.qiniu.com/dora/manual/1291/persistent-data-processing-pfop
+ 与 template_id 二选一
force: int or str, optional
强制执行持久化处理开关
persistent_type: int or str, optional
持久化处理类型，为 '1' 时开启闲时任务
+ template_id: str, optional
+ 与 fops 二选一
Returns
-------
ret: dict
持久化处理的 persistentId，类似 {"persistentId": 5476bedf7823de4068253bae};
resp: ResponseInfo
"""
- ops = ';'.join(fops)
- data = {'bucket': self.bucket, 'key': key, 'fops': ops}
+ if not fops and not workflow_template_id:
+ raise ValueError('Must provide one of fops or template_id')
+ data = {
+ 'bucket': self.bucket,
+ 'key': key,
+ }
if self.pipeline:
data['pipeline'] = self.pipeline
if self.notify_url:
data['notifyURL'] = self.notify_url
+ if fops:
+ data['fops'] = ';'.join(fops)
if force == 1 or force == '1':
data['force'] = str(force)
if persistent_type and type(int(persistent_type)) is int:
data['type'] = str(persistent_type)
+ if workflow_template_id:
+ data['workflowTemplateID'] = workflow_template_id
url = '{0}/pfop'.format(config.get_default('default_api_host'))
return http._post_with_auth(url, data, self.auth)
diff --git a/tests/cases/test_services/test_processing/test_pfop.py b/tests/cases/test_services/test_processing/test_pfop.py
index 003be43f..ebaf18f4 100644
--- a/tests/cases/test_services/test_processing/test_pfop.py
+++ b/tests/cases/test_services/test_processing/test_pfop.py
@@ -1,6 +1,5 @@
import pytest
-
from qiniu import PersistentFop, op_save
@@ -16,6 +15,7 @@ def test_pfop_execute(self, qn_auth):
]
ret, resp = pfop.execute('sintel_trailer.mp4', ops, 1)
assert resp.status_code == 200, resp
+ assert ret is not None, resp
assert ret['persistentId'] is not None, resp
global persistent_id
persistent_id = ret['persistentId']
@@ -27,23 +27,71 @@ def test_pfop_get_status(self, qn_auth):
assert resp.status_code == 200, resp
assert ret is not None, resp
- def test_pfop_idle_time_task(self, set_conf_default, qn_auth):
- persistence_key = 'python-sdk-pfop-test/test-pfop-by-api'
+ @pytest.mark.parametrize(
+ 'persistent_options',
+ (
+ # included by above test_pfop_execute
+ # {
+ # 'persistent_type': None,
+ # },
+ {
+ 'persistent_type': 0,
+ },
+ {
+ 'persistent_type': 1,
+ },
+ {
+ 'workflow_template_id': 'test-workflow',
+ },
+ )
+ )
+ def test_pfop_idle_time_task(
+ self,
+ set_conf_default,
+ qn_auth,
+ bucket_name,
+ persistent_options,
+ ):
+ persistent_type = persistent_options.get('persistent_type')
+ workflow_template_id = persistent_options.get('workflow_template_id', None)
+
+ execute_opts = {}
+ if workflow_template_id:
+ execute_opts['workflow_template_id'] = workflow_template_id
+ else:
+ persistent_key = '_'.join([
+ 'test-pfop/test-pfop-by-api',
+ 'type',
+ str(persistent_type)
+ ])
+ execute_opts['fops'] = [
+ op_save(
+ op='avinfo',
+ bucket=bucket_name,
+ key=persistent_key
+ )
+ ]
+
+ if persistent_type is not None:
+ execute_opts['persistent_type'] = persistent_type
+
+ pfop = PersistentFop(qn_auth, bucket_name)
+ key = 'qiniu.png'
+ ret, resp = pfop.execute(
+ key,
+ **execute_opts
+ )
- key = 'sintel_trailer.mp4'
- pfop = PersistentFop(qn_auth, 'testres')
- ops = [
- op_save(
- op='avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240',
- bucket='pythonsdk',
- key=persistence_key
- )
- ]
- ret, resp = pfop.execute(key, ops, force=1, persistent_type=1)
assert resp.status_code == 200, resp
+ assert ret is not None
assert 'persistentId' in ret, resp
ret, resp = pfop.get_status(ret['persistentId'])
assert resp.status_code == 200, resp
- assert ret['type'] == 1, resp
+ assert ret is not None
assert ret['creationDate'] is not None, resp
+
+ if persistent_id == 1:
+ assert ret['type'] == 1, resp
+ elif workflow_template_id:
+ assert workflow_template_id in ret['taskFrom'], resp
diff --git a/tests/cases/test_services/test_storage/test_upload_pfop.py b/tests/cases/test_services/test_storage/test_upload_pfop.py
index 78818ba4..3effa9c7 100644
--- a/tests/cases/test_services/test_storage/test_upload_pfop.py
+++ b/tests/cases/test_services/test_storage/test_upload_pfop.py
@@ -12,36 +12,60 @@
# or this test will continue to occupy bucket space.
class TestPersistentFopByUpload:
@pytest.mark.parametrize('temp_file', [10 * MB], indirect=True)
- @pytest.mark.parametrize('persistent_type', [None, 0, 1])
+ @pytest.mark.parametrize(
+ 'persistent_options',
+ (
+ {
+ 'persistent_type': None,
+ },
+ {
+ 'persistent_type': 0,
+ },
+ {
+ 'persistent_type': 1,
+ },
+ {
+ 'persistent_workflow_template_id': 'test-workflow',
+ },
+ )
+ )
def test_pfop_with_upload(
self,
set_conf_default,
qn_auth,
bucket_name,
temp_file,
- persistent_type
+ persistent_options,
):
- key = 'test-pfop-upload-file'
- persistent_key = '_'.join([
- 'test-pfop-by-upload',
- 'type',
- str(persistent_type)
- ])
- persistent_ops = ';'.join([
- qiniu.op_save(
- op='avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240',
- bucket=bucket_name,
- key=persistent_key
- )
- ])
+ key = 'test-pfop/upload-file'
+ persistent_type = persistent_options.get('persistent_type')
+ persistent_workflow_template_id = persistent_options.get('persistent_workflow_template_id')
+
+ upload_policy = {}
- upload_policy = {
- 'persistentOps': persistent_ops
- }
+ # set pfops or tmplate id
+ if persistent_workflow_template_id:
+ upload_policy['persistentWorkflowTemplateID'] = persistent_workflow_template_id
+ else:
+ persistent_key = '_'.join([
+ 'test-pfop/test-pfop-by-upload',
+ 'type',
+ str(persistent_type)
+ ])
+ persistent_ops = ';'.join([
+ qiniu.op_save(
+ op='avinfo',
+ bucket=bucket_name,
+ key=persistent_key
+ )
+ ])
+ upload_policy['persistentOps'] = persistent_ops
+ # set persistent type
if persistent_type is not None:
upload_policy['persistentType'] = persistent_type
+ # upload
token = qn_auth.upload_token(
bucket_name,
key,
@@ -61,6 +85,10 @@ def test_pfop_with_upload(
pfop = qiniu.PersistentFop(qn_auth, bucket_name)
ret, resp = pfop.get_status(ret['persistentId'])
assert resp.status_code == 200, resp
+ assert ret is not None, resp
+ assert ret['creationDate'] is not None, resp
+
if persistent_type == 1:
assert ret['type'] == 1, resp
- assert ret['creationDate'] is not None, resp
+ elif persistent_workflow_template_id:
+ assert persistent_workflow_template_id in ret['taskFrom'], resp
From 945075b190dfd0a5da03d40cb0345d721b5825c7 Mon Sep 17 00:00:00 2001
From: lihsai0 <lihsai0@gmail.com>
Date: Wed, 16 Oct 2024 09:39:48 +0800
Subject: [PATCH 5/5] update version to 7.15.0 and changelog
---
CHANGELOG.md | 8 +++++---
qiniu/__init__.py | 2 +-
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6bf26f46..5722b56b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,8 @@
# Changelog
+## 7.15.0
+* 对象存储，持久化处理支持工作流模版
+* 对象存储，修复 Windows 平台兼容性问题
+
## 7.14.0
* 对象存储，空间管理、上传文件新增备用域名重试逻辑
* 对象存储，调整查询区域主备域名
@@ -44,7 +48,7 @@
## 7.9.0（2022-07-20）
* 对象存储，支持使用时不配置区域信息，SDK 自动获取；
* 对象存储，新增 list_domains API 用于查询空间绑定的域名
-* 对象存储，上传 API 新增支持设置自定义元数据，详情见 put_data, put_file, put_stream API
+* 对象存储，上传 API 新增支持设置自定义元数据，详情见 put_data, put_file, put_stream API
* 解决部分已知问题
## 7.8.0（2022-06-08）
@@ -237,5 +241,3 @@
* 代码覆盖度报告
* policy改为dict, 便于灵活增加，并加入过期字段检查
* 文件列表支持目录形式
-
-
diff --git a/qiniu/__init__.py b/qiniu/__init__.py
index 55acfb25..1ae68c00 100644
--- a/qiniu/__init__.py
+++ b/qiniu/__init__.py
@@ -9,7 +9,7 @@
# flake8: noqa
-__version__ = '7.14.0'
+__version__ = '7.15.0'
from .auth import Auth, QiniuMacAuth