Skip to content

Commit bc7cf1f

Browse files
bajdzunnick-zh
andauthored
MPM-296 Improve avro checks (#27)
* Docker changes * Makefile changes * Bump deps * Replace deprecated method * Bump orbs * Fix phpstan errors * Remove var_dump * Added covers annotation * Change infection MSI * Check default type * Added type map * Refactor logic * Fix coverage * Change msi * Changed logic for type checking * Show invalid fields on output * Cleanup * Added good nested schema * Clean up * Revert avro check command * Added new check command for default type * Fixed phpstan error * Cleanup * Resolve discussions * Cleanup * Update tests/Command/CheckAllSchemaTemplatesDefaultTypeCommandTest.php Co-authored-by: Marko <[email protected]> Co-authored-by: Nick <[email protected]>
1 parent 3958a09 commit bc7cf1f

35 files changed

+638
-99
lines changed

.circleci/config.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
version: 2.1
22

33
orbs:
4-
ci-caching: jobcloud/ci-caching@0.12
5-
ci-php: jobcloud/ci-php@0.29
4+
ci-caching: jobcloud/ci-caching@1.0.2
5+
ci-php: jobcloud/ci-php@0.32
66

77
workflows:
88
test-console-kafka-schema-registry:
@@ -33,4 +33,4 @@ workflows:
3333
dockerComposeFile: "./docker/docker-compose.yml"
3434
dependencyCheckSumFile: "./composer.json"
3535
requires:
36-
- ci-php/install-dependencies
36+
- ci-php/install-dependencies

Makefile

+14-4
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
.PHONY: clean fix-code-style test coverage install-dependencies code-style static-analysis xdebug-disable xdebug-enable update-dependencies
1+
.PHONY: clean code-check fix-code-style test coverage help install-dependencies code-style static-analysis ci-static-analysis infection-testing pcov-disable pcov-enable update-dependencies
22
.DEFAULT_GOAL := test
33

44
INFECTION = ./vendor/bin/infection
@@ -9,7 +9,7 @@ PHPCBF = ./vendor/bin/phpcbf
99
COVCHK = ./vendor/bin/coverage-check
1010

1111
clean:
12-
rm -rf ./vendor
12+
rm -rf ./vendor ./build
1313

1414
code-check:
1515
${PHPCS}
@@ -32,22 +32,30 @@ ci-static-analysis:
3232
${PHPSTAN} analyse
3333

3434
test:
35-
${PHPUNIT}
35+
${PHPUNIT} --no-coverage
3636

3737
coverage:
3838
${PHPUNIT} && ${COVCHK} build/logs/phpunit/coverage/coverage.xml 100
3939

4040
infection-testing:
4141
make coverage
4242
cp -f build/logs/phpunit/junit.xml build/logs/phpunit/coverage/junit.xml
43-
${INFECTION} --coverage=build/logs/phpunit/coverage --min-msi=82 --threads=`nproc`
43+
sudo php-ext-disable pcov
44+
${INFECTION} --coverage=build/logs/phpunit/coverage --min-msi=84 --threads=`nproc`
45+
sudo php-ext-enable pcov
4446

4547
install-dependencies:
4648
composer install
4749

4850
update-dependencies:
4951
composer update
5052

53+
pcov-enable:
54+
sudo php-ext-enable pcov
55+
56+
pcov-disable:
57+
sudo php-ext-disable pcov
58+
5159
help:
5260
# Usage:
5361
# make <target> [OPTION=value]
@@ -64,3 +72,5 @@ help:
6472
# static-analysis Run static analysis using phpstan
6573
# ci-static-analysis Run static analysis using phpstan for CI only.
6674
# test Run tests
75+
# pcov-enable Enable pcov
76+
# pcov-disable Disable pcov

composer.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@
2222
"ext-json": "*"
2323
},
2424
"require-dev": {
25-
"phpunit/phpunit": "^8.3",
26-
"phpstan/phpstan": "^0.11.15",
27-
"infection/infection": "^0.15.0",
25+
"phpunit/phpunit": "^9.5",
26+
"phpstan/phpstan": "^0.12",
27+
"infection/infection": "^0.18",
2828
"squizlabs/php_codesniffer": "^3.4",
2929
"rregeer/phpunit-coverage-check": "^0.3.1"
3030
},

docker/dev/php/Dockerfile

+5-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1-
FROM php:7.3-cli-alpine3.10
1+
FROM php:7.3-cli-alpine3.13
22

33
ARG HOST_USER_ID
44

55
COPY files/bin/ /usr/local/bin/
6+
COPY files/php/ /phpIni
67

78
# SYS: Install required packages
89
RUN apk --no-cache upgrade && \
@@ -22,6 +23,9 @@ RUN /bin/bash -c 'if [ -n "$HOST_USER_ID" ] && [ "$HOST_USER_ID" -lt 60000 ]; th
2223
usermod -u ${HOST_USER_ID} www-data; \
2324
fi'
2425

26+
RUN usermod -s /bin/bash www-data && \
27+
echo 'www-data ALL=(ALL) NOPASSWD: ALL' > '/etc/sudoers.d/www-data'
28+
2529
COPY files/user-home /home/www-data
2630

2731
# SYS: add ssh config (for GitHub)

docker/dev/php/files/php/20-pcov.ini

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
extension=pcov.so

phpstan.neon

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
parameters:
2-
level: 7
2+
level: 8
33
paths: [ src ]

phpunit.xml

+37-44
Original file line numberDiff line numberDiff line change
@@ -1,45 +1,38 @@
11
<?xml version="1.0" encoding="UTF-8"?>
2-
3-
<phpunit bootstrap = "vendor/autoload.php"
4-
backupGlobals = "false"
5-
backupStaticAttributes = "false"
6-
colors = "true"
7-
convertErrorsToExceptions = "true"
8-
convertNoticesToExceptions = "true"
9-
convertWarningsToExceptions = "true"
10-
processIsolation = "false"
11-
stopOnFailure = "false">
12-
13-
<php>
14-
<ini name="max_execution_time" value="-1"/>
15-
<ini name="html_errors" value="false"/>
16-
<ini name="memory_limit" value="1G"/>
17-
18-
<ini name="xdebug.default_enable" value="1" />
19-
<ini name="xdebug.enable_coverage" value="1" />
20-
<ini name="xdebug.remote_autostart" value="0" />
21-
<ini name="xdebug.remote_enable" value="0" />
22-
<ini name="xdebug.overload_var_dump" value="0" />
23-
<ini name="xdebug.show_mem_delta" value="0" />
24-
</php>
25-
26-
<testsuites>
27-
<testsuite name="Schema Console Registry Commands Test Suite">
28-
<directory>./tests</directory>
29-
</testsuite>
30-
</testsuites>
31-
32-
<logging>
33-
<log type="coverage-text" target="php://stdout" showOnlySummary="true"/>
34-
<log type="coverage-html" target="build/logs/phpunit/coverage"/>
35-
<log type="coverage-xml" target="build/logs/phpunit/coverage/coverage-xml"/>
36-
<log type="coverage-clover" target="build/logs/phpunit/coverage/coverage.xml"/>
37-
<log type="junit" target="build/logs/phpunit/junit.xml"/>
38-
</logging>
39-
40-
<filter>
41-
<whitelist>
42-
<directory suffix=".php">./src</directory>
43-
</whitelist>
44-
</filter>
45-
</phpunit>
2+
<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
bootstrap="vendor/autoload.php"
4+
backupGlobals="false"
5+
backupStaticAttributes="false"
6+
colors="true"
7+
convertErrorsToExceptions="true"
8+
convertNoticesToExceptions="true"
9+
convertWarningsToExceptions="true"
10+
processIsolation="false"
11+
stopOnFailure="false"
12+
forceCoversAnnotation="true"
13+
xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/9.5/phpunit.xsd">
14+
<coverage>
15+
<include>
16+
<directory suffix=".php">./src</directory>
17+
</include>
18+
<report>
19+
<clover outputFile="build/logs/phpunit/coverage/coverage.xml"/>
20+
<html outputDirectory="build/logs/phpunit/coverage"/>
21+
<text outputFile="php://stdout" showOnlySummary="true"/>
22+
<xml outputDirectory="build/logs/phpunit/coverage/coverage-xml"/>
23+
</report>
24+
</coverage>
25+
<php>
26+
<ini name="max_execution_time" value="-1"/>
27+
<ini name="html_errors" value="false"/>
28+
<ini name="memory_limit" value="1G"/>
29+
</php>
30+
<testsuites>
31+
<testsuite name="Schema Console Registry Commands Test Suite">
32+
<directory>./tests</directory>
33+
</testsuite>
34+
</testsuites>
35+
<logging>
36+
<junit outputFile="build/logs/phpunit/junit.xml"/>
37+
</logging>
38+
</phpunit>
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
<?php
2+
3+
namespace Jobcloud\SchemaConsole\Command;
4+
5+
use GuzzleHttp\Exception\RequestException;
6+
use Jobcloud\SchemaConsole\Helper\SchemaFileHelper;
7+
use Symfony\Component\Console\Command\Command;
8+
use Symfony\Component\Console\Input\InputArgument;
9+
use Symfony\Component\Console\Input\InputInterface;
10+
use Symfony\Component\Console\Output\OutputInterface;
11+
use Symfony\Component\Console\Style\SymfonyStyle;
12+
13+
class CheckAllSchemaTemplatesDefaultTypeCommand extends Command
14+
{
15+
private const TYPE_MAP = [
16+
"null" => "null",
17+
"boolean" => "boolean",
18+
"integer" => "int",
19+
"string" => "string",
20+
"double" => "double",
21+
"array" => "array",
22+
];
23+
24+
/**
25+
* @return void
26+
*/
27+
protected function configure(): void
28+
{
29+
$this
30+
->setName('kafka-schema-registry:check:template:default:type:all')
31+
->setDescription('Checks for default value type for all schema templates in folder')
32+
->setHelp('Checks for default value type for all schema templates in folder')
33+
->addArgument(
34+
'schemaTemplateDirectory',
35+
InputArgument::REQUIRED,
36+
'Path to avro schema template directory'
37+
);
38+
}
39+
40+
/**
41+
* @param InputInterface $input
42+
* @param OutputInterface $output
43+
* @return integer
44+
* @throws RequestException
45+
*/
46+
public function execute(InputInterface $input, OutputInterface $output): int
47+
{
48+
/** @var string $directory */
49+
$directory = $input->getArgument('schemaTemplateDirectory');
50+
$avroFiles = SchemaFileHelper::getAvroFiles($directory);
51+
52+
$io = new SymfonyStyle($input, $output);
53+
54+
$failed = [];
55+
56+
if (false === $this->checkSchemas($avroFiles, $failed)) {
57+
$io->error('Following schema templates have invalid default value types:');
58+
$io->listing($failed);
59+
60+
return 1;
61+
}
62+
63+
$io->success('All schema templates have valid default value types');
64+
65+
return 0;
66+
}
67+
68+
/**
69+
* @param array<string, mixed> $avroFiles
70+
* @param array<string, mixed> $failed
71+
* @return boolean
72+
*/
73+
private function checkSchemas(array $avroFiles, array &$failed = []): bool
74+
{
75+
$failed = [];
76+
77+
foreach ($avroFiles as $schemaName => $avroFile) {
78+
/** @var string $localSchema */
79+
$localSchema = file_get_contents($avroFile);
80+
81+
$invalidFields = $this->checkDefaultType($localSchema);
82+
83+
foreach ($invalidFields as $invalidField) {
84+
$failed[] = $invalidField;
85+
}
86+
}
87+
88+
return 0 === count($failed);
89+
}
90+
91+
/**
92+
* @param string $localSchema
93+
* @return array<string, mixed>
94+
*/
95+
private function checkDefaultType(string $localSchema): array
96+
{
97+
$decodedSchema = json_decode($localSchema);
98+
if (!property_exists($decodedSchema, 'fields')) {
99+
return [];
100+
}
101+
102+
return $this->checkAllFields($decodedSchema);
103+
}
104+
105+
/**
106+
* @param mixed $decodedSchema
107+
* @param array<mixed, mixed> $defaultFields
108+
* @return array<string, mixed>
109+
*/
110+
private function checkAllFields($decodedSchema, array $defaultFields = []): array
111+
{
112+
foreach ($decodedSchema->fields as $field) {
113+
if (!property_exists($field, 'default')) {
114+
continue;
115+
}
116+
117+
$defaultFields[$field->name] = $this->getFieldName($decodedSchema, $field);
118+
119+
$fieldTypes = $field->type;
120+
121+
if (!is_array($fieldTypes)) {
122+
$fieldTypes = [$fieldTypes];
123+
}
124+
125+
foreach ($fieldTypes as $fieldType) {
126+
$defaultFields = $this->checkSingleField($fieldType, $field, $defaultFields);
127+
}
128+
}
129+
130+
return $defaultFields;
131+
}
132+
133+
/**
134+
* @param mixed $fieldType
135+
* @param mixed $field
136+
* @param array<mixed, mixed> $defaultFields
137+
* @return array<string, mixed>
138+
*/
139+
private function checkSingleField($fieldType, $field, array $defaultFields): array
140+
{
141+
$defaultType = strtolower(gettype($field->default));
142+
143+
if (is_string($fieldType)) {
144+
if (
145+
self::TYPE_MAP[$defaultType] === $fieldType
146+
|| $this->isContainedInBiggerType(self::TYPE_MAP[$defaultType], $fieldType)
147+
) {
148+
unset($defaultFields[$field->name]);
149+
}
150+
}
151+
152+
if (property_exists($fieldType, 'type') && $fieldType->type === 'array') {
153+
if (is_string($defaultType) && self::TYPE_MAP[$defaultType] === $fieldType->type) {
154+
unset($defaultFields[$field->name]);
155+
}
156+
}
157+
158+
return $defaultFields;
159+
}
160+
161+
/**
162+
* @param string $defaultType
163+
* @param string $currentType
164+
* @return bool
165+
*/
166+
private function isContainedInBiggerType(string $defaultType, string $currentType): bool
167+
{
168+
if ($currentType === 'double' && ($defaultType === 'int' || $defaultType === 'float')) {
169+
return true;
170+
}
171+
172+
if ($currentType === 'float' && $defaultType === 'int') {
173+
return true;
174+
}
175+
176+
return false;
177+
}
178+
179+
/**
180+
* @param mixed $decodedSchema
181+
* @param mixed $field
182+
* @return string
183+
*/
184+
private function getFieldName($decodedSchema, $field): string
185+
{
186+
return $decodedSchema->namespace . '.' . $decodedSchema->name . '.' . $field->name;
187+
}
188+
}

src/Command/CheckAllSchemaTemplatesDocCommentsCommand.php

+2-2
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@ public function execute(InputInterface $input, OutputInterface $output): int
6363

6464

6565
/**
66-
* @param array $avroFiles
67-
* @param array $failed
66+
* @param array<string, mixed> $avroFiles
67+
* @param array<string, mixed> $failed
6868
* @return boolean
6969
*/
7070
private function checkDocCommentsOnSchemaTemplates(array $avroFiles, array &$failed = []): bool

0 commit comments

Comments
 (0)