-
Notifications
You must be signed in to change notification settings - Fork 129
internal/testrunner/script: add script testing package #3012
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
ab30d9e
9d132cf
9825fc9
09e3c8b
f72e96c
0da1133
7588561
819f814
a524978
fc8c9db
054ae48
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -97,6 +97,9 @@ test-check-packages-with-kind: | |||||
| test-check-packages-other: | ||||||
| PACKAGE_TEST_TYPE=other ./scripts/test-check-packages.sh | ||||||
|
|
||||||
| test-check-packages-independent-script: | ||||||
| elastic-package test script -C test/packages/other/with_script --external-stack=false --defer-cleanup 1s | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would create a new folder for this test package like The reasoning of this is that currently, this
So, I would create a new folder and update this target:
Suggested change
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
The twice use of this is intentional. |
||||||
|
|
||||||
| test-check-packages-false-positives: | ||||||
| PACKAGE_TEST_TYPE=false_positives ./scripts/test-check-false-positives.sh | ||||||
|
|
||||||
|
|
@@ -133,7 +136,7 @@ test-profiles-command: | |||||
| test-check-update-version: | ||||||
| ./scripts/test-check-update-version.sh | ||||||
|
|
||||||
| test: test-go test-stack-command test-check-packages test-profiles-command test-build-install-zip test-build-zip test-build-install-zip-file test-build-install-zip-file-shellinit test-check-update-version test-profiles-command test-system-test-flags | ||||||
| test: test-go test-stack-command test-check-packages test-check-packages-independent-script test-profiles-command test-build-install-zip test-build-zip test-build-install-zip-file test-build-install-zip-file-shellinit test-check-update-version test-profiles-command test-system-test-flags | ||||||
|
|
||||||
| check-git-clean: | ||||||
| git update-index --really-refresh | ||||||
|
|
||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -31,6 +31,7 @@ import ( | |
| "github.com/elastic/elastic-package/internal/testrunner/runners/policy" | ||
| "github.com/elastic/elastic-package/internal/testrunner/runners/static" | ||
| "github.com/elastic/elastic-package/internal/testrunner/runners/system" | ||
| "github.com/elastic/elastic-package/internal/testrunner/script" | ||
| ) | ||
|
|
||
| const testLongDescription = `Use this command to run tests on a package. Currently, the following types of tests are available: | ||
|
|
@@ -95,6 +96,9 @@ func setupTestCommand() *cobraext.Command { | |
| systemCmd := getTestRunnerSystemCommand() | ||
| cmd.AddCommand(systemCmd) | ||
|
|
||
| scriptCmd := getTestRunnerScriptCommand() | ||
| cmd.AddCommand(scriptCmd) | ||
|
|
||
| policyCmd := getTestRunnerPolicyCommand() | ||
| cmd.AddCommand(policyCmd) | ||
|
|
||
|
|
@@ -600,6 +604,46 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error { | |
| return nil | ||
| } | ||
|
|
||
| func getTestRunnerScriptCommand() *cobra.Command { | ||
| cmd := &cobra.Command{ | ||
| Use: "script", | ||
| Short: "Run script tests", | ||
| Long: "Run script tests for the package.", | ||
| Args: cobra.NoArgs, | ||
| RunE: testRunnerScriptCommandAction, | ||
| } | ||
|
|
||
| cmd.Flags().String(cobraext.ScriptsFlagName, "", cobraext.ScriptsFlagDescription) | ||
| cmd.Flags().Bool(cobraext.ExternalStackFlagName, true, cobraext.ExternalStackFlagDescription) | ||
| cmd.Flags().StringSliceP(cobraext.DataStreamsFlagName, "d", nil, cobraext.DataStreamsFlagDescription) | ||
| cmd.Flags().String(cobraext.RunPatternFlagName, "", cobraext.RunPatternFlagDescription) | ||
| cmd.Flags().BoolP(cobraext.UpdateScriptTestArchiveFlagName, "u", false, cobraext.UpdateScriptTestArchiveFlagDescription) | ||
| cmd.Flags().BoolP(cobraext.WorkScriptTestFlagName, "w", false, cobraext.WorkScriptTestFlagDescription) | ||
| cmd.Flags().Bool(cobraext.ContinueOnErrorFlagName, false, cobraext.ContinueOnErrorFlagDescription) | ||
| cmd.Flags().Bool(cobraext.VerboseScriptFlagName, false, cobraext.VerboseScriptFlagDescription) | ||
|
|
||
| cmd.MarkFlagsMutuallyExclusive(cobraext.DataStreamsFlagName, cobraext.DataStreamsFlagName) | ||
|
|
||
| return cmd | ||
| } | ||
|
|
||
| func testRunnerScriptCommandAction(cmd *cobra.Command, args []string) error { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The logic in this function is rudimentary and only intended to allow an MVP to be demonstrated. In order to be able to render reports and redirect to file output this needs enhancement. |
||
| cmd.Println("Run script tests for the package") | ||
| pkgRoot, err := packages.FindPackageRoot() | ||
| if err != nil { | ||
| if err == packages.ErrPackageRootNotFound { | ||
| return errors.New("package root not found") | ||
| } | ||
| return fmt.Errorf("locating package root failed: %w", err) | ||
| } | ||
| pkg := filepath.Base(pkgRoot) | ||
| cmd.Printf("--- Test results for package: %s - START ---\n", pkg) | ||
| err = script.Run(cmd.OutOrStderr(), cmd, args) | ||
| cmd.Printf("--- Test results for package: %s - END ---\n", pkg) | ||
| cmd.Println("Done") | ||
| return err | ||
| } | ||
|
|
||
| func getTestRunnerPolicyCommand() *cobra.Command { | ||
| cmd := &cobra.Command{ | ||
| Use: "policy", | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,209 @@ | ||||||||||||||||||||||||||||||
| # HOWTO: Writing script tests for a package | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| Script testing is an advanced topic that assumes knowledge of [pipeline](./pipeline_testing.md) | ||||||||||||||||||||||||||||||
| and [system](./system_testing.md) testing. | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| Testing packages with script testing is only intended for testing cases that | ||||||||||||||||||||||||||||||
| cannot be adequately covered by the pipeline and system testing tools such as | ||||||||||||||||||||||||||||||
| testing failure paths and package upgrades. It can also be used for debugging | ||||||||||||||||||||||||||||||
| integrations stack issues. | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| ## Introduction | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| The script testing system is built on the Go testscript package with extensions | ||||||||||||||||||||||||||||||
| provided to allow scripting of stack and integration operations such as | ||||||||||||||||||||||||||||||
| bringing up a stack, installing packages and running agents. For example, using | ||||||||||||||||||||||||||||||
| these commands it is possible to express a system test as described in the | ||||||||||||||||||||||||||||||
| system testing [Conceptual Process](./system_testing.md#conceptual-process) section. | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| ## Expressing tests | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| Tests are written as [txtar format](https://pkg.go.dev/golang.org/x/tools/txtar#hdr-Txtar_format) | ||||||||||||||||||||||||||||||
| files in a data stream's \_dev/test/scripts directory. The logic for the test is | ||||||||||||||||||||||||||||||
| written in the txtar file's initial comment section and any additional resource | ||||||||||||||||||||||||||||||
| files are included in the txtar file's files sections. | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| The standard commands and behaviors for testscript scripts are documented in | ||||||||||||||||||||||||||||||
| the [testscript package documentation](https://pkg.go.dev/github.com/rogpeppe/go-internal/testscript). | ||||||||||||||||||||||||||||||
|
Comment on lines
+22
to
+28
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Just for the record. Could you comment, maybe in the description of the PR, why you chose There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It's the only well maintained and extensively used script testing package available with the features that we need. It's not a language, so I'm not sure what competing systems you are referring to. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
I see it as a great tool for black-box testing of commands. Here we are exposing many functions as commands, that could stay as functions if we were using some embedded scripting language.
Well, I can agree that it is not a turing-complete language, but it is something that along with txtar is used to express scripts, what could very well called language, but I guess this is a semantic discussion 🙂
There are other embeddable systems that can be used to express scripts, from goja to CEL, starlark, or even a yaml list. I agree with the choice, only that it wouldn't have been the first option I would have thought of for this, and I would like to have the decision stored in some place. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I've added it to the issue that this is addressing. |
||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| ## Extension commands | ||||||||||||||||||||||||||||||
jsoriano marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| The test script command provides additional commands to aid in interacting with | ||||||||||||||||||||||||||||||
| a stack, starting agents and services and validating results. | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| - `sleep <duration>`: sleep for a duration (Go `time.Duration` parse syntax) | ||||||||||||||||||||||||||||||
| - `date [<ENV_VAR_NAME>]`: print the current time in RFC3339, optionally setting a variable with the value | ||||||||||||||||||||||||||||||
| - `GET [-json] <url>`: perform an HTTP GET request, emitting the response body to stdout and optionally formatting indented JSON | ||||||||||||||||||||||||||||||
| - `POST [-json] [-content <content-type>] <body-path> <url>`: perform an HTTP POST request, emitting the response body to stdout and optionally formatting indented JSON | ||||||||||||||||||||||||||||||
| - `match_file <pattern_file_path> <data_path>`: perform a grep pattern match between a pattern file and a data file | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| - stack commands: | ||||||||||||||||||||||||||||||
| - `stack_up [-profile <profile>] [-provider <provider>] [-timeout <duration>] <stack-version>`: bring up a version of the Elastic stack | ||||||||||||||||||||||||||||||
| - `use_stack [-profile <profile>] [-timeout <duration>]`: use a running Elastic stack | ||||||||||||||||||||||||||||||
| - `stack_down [-profile <profile>] [-provider <provider>] [-timeout <duration>]`: take down a started Elastic stack | ||||||||||||||||||||||||||||||
| - `dump_logs [-profile <profile>] [-provider <provider>] [-timeout <duration>] [-since <RFC3339 time>] [<dirpath>]`: dump the logs from the stack into a directory | ||||||||||||||||||||||||||||||
| - `get_policy [-profile <profile>] [-timeout <duration>] <policy_name>`: print the details for a policy | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| - agent commands: | ||||||||||||||||||||||||||||||
| - `install_agent [-profile <profile>] [-timeout <duration>] [<network_name_label>]`: install an Elastic Agent policy, setting the environment variable named in the positional argument | ||||||||||||||||||||||||||||||
| - `uninstall_agent [-profile <profile>] [-timeout <duration>]`: remove an installed Elastic Agent policy | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| - package commands: | ||||||||||||||||||||||||||||||
| - `add_package [-profile <profile>] [-timeout <duration>]`: add the current package's assets | ||||||||||||||||||||||||||||||
| - `remove_package [-profile <profile>] [-timeout <duration>]`: remove assets for the current package | ||||||||||||||||||||||||||||||
| - `add_package_zip [-profile <profile>] [-timeout <duration>] <path_to_zip>`: add assets from a Zip-packaged integration package | ||||||||||||||||||||||||||||||
| - `remove_package_zip [-profile <profile>] [-timeout <duration>] <path_to_zip>`: remove assets for Zip-packaged integration package | ||||||||||||||||||||||||||||||
| - `upgrade_package_latest [-profile <profile>] [-timeout <duration>] [<package_name>]`: upgrade the current package or another named package to the latest version | ||||||||||||||||||||||||||||||
| - `add_package_policy [-profile <profile>] [-timeout <duration>] [-policy <policy_name>] <config.yaml> <name_var_label>`: add a package policy, setting the environment variable named in the positional argument | ||||||||||||||||||||||||||||||
| - `remove_package_policy [-profile <profile>] [-timeout <duration>] <data_stream_name>`: remove a package policy | ||||||||||||||||||||||||||||||
| - `get_docs [-profile <profile>] [-timeout <duration>] [<data_stream>]`: get documents from a data stream | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| - docker commands: | ||||||||||||||||||||||||||||||
| - `docker_up [-profile <profile>] [-timeout <duration>] <dir>`: start a docker service defined in the provided directory | ||||||||||||||||||||||||||||||
| - `docker_down [-timeout <duration>] <name>`: stop a started docker service and print the docker logs to stdout | ||||||||||||||||||||||||||||||
| - `docker_signal [-timeout <duration>] <name> <signal>`: send a signal to a running docker service | ||||||||||||||||||||||||||||||
| - `docker_wait_exit [-timeout <duration>] <name>`: wait for a docker service to exit | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| - pipeline commands: | ||||||||||||||||||||||||||||||
| - `install_pipelines [-profile <profile>] [-timeout <duration>] <path_to_data_stream>`: install ingest pipelines from a path | ||||||||||||||||||||||||||||||
| - `simulate [-profile <profile>] [-timeout <duration>] <path_to_data_stream> <pipeline> <path_to_data>`: run a pipeline test, printing the result as pretty-printed JSON to standard output | ||||||||||||||||||||||||||||||
| - `uninstall_pipelines [-profile <profile>] [-timeout <duration>] <path_to_data_stream>`: remove installed ingest pipelines | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| ## Environment variables | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| - `PROFILE`: the `elastic-package` profile being used | ||||||||||||||||||||||||||||||
| - `CONFIG_ROOT`: the `elastic-package` configuration root path | ||||||||||||||||||||||||||||||
| - `CONFIG_PROFILES`: the `elastic-package` profiles configuration root path | ||||||||||||||||||||||||||||||
| - `HOME`: the user's home directory path | ||||||||||||||||||||||||||||||
| - `PACKAGE_NAME`: the name of the running package | ||||||||||||||||||||||||||||||
| - `PACKAGE_BASE`: the basename of the path to the root of the running package | ||||||||||||||||||||||||||||||
| - `PACKAGE_ROOT`: the path to the root of the running package | ||||||||||||||||||||||||||||||
| - `CURRENT_VERSION`: the current version of the package | ||||||||||||||||||||||||||||||
| - `PREVIOUS_VERSION`: the previous version of the package | ||||||||||||||||||||||||||||||
| - `DATA_STREAM`: the name of the data stream | ||||||||||||||||||||||||||||||
| - `DATA_STREAM_ROOT`: the path to the root of the data stream | ||||||||||||||||||||||||||||||
jsoriano marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| ## Conditions | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| The testscript package allows conditions to be set that allow conditional | ||||||||||||||||||||||||||||||
| execution of commands. The test script command adds a condition that reflects | ||||||||||||||||||||||||||||||
| the state of the `--external-stack` flag. This allows tests to be written that | ||||||||||||||||||||||||||||||
| conditionally use either an externally managed stack, or a stack that has been | ||||||||||||||||||||||||||||||
| started by the test script. | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| ## Example | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| As an example, a basic system test could be expressed as follows. | ||||||||||||||||||||||||||||||
| ``` | ||||||||||||||||||||||||||||||
| # Only run the test if --external-stack=true. | ||||||||||||||||||||||||||||||
| [!external_stack] skip 'Skipping external stack test.' | ||||||||||||||||||||||||||||||
| # Only run the test if the jq executable is in $PATH. This is needed for a test below. | ||||||||||||||||||||||||||||||
| [!exec:jq] skip 'Skipping test requiring absent jq command' | ||||||||||||||||||||||||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It looks like several sample tests here depend on This would help to avoid depending on external tools that may not be installed, specially on Windows. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If it becomes necessary, those can be added. Both There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Agree this can be added later. I am not so worried about the current CI, but about future deployments where we might forget to install
|
||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| # Register running stack. | ||||||||||||||||||||||||||||||
| use_stack -profile ${CONFIG_PROFILES}/${PROFILE} | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| # Install an agent. | ||||||||||||||||||||||||||||||
| install_agent -profile ${CONFIG_PROFILES}/${PROFILE} NETWORK_NAME | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| # Bring up a docker container. | ||||||||||||||||||||||||||||||
| # | ||||||||||||||||||||||||||||||
| # The service is described in the test-hits/docker-compose.yml below with | ||||||||||||||||||||||||||||||
| # its logs in test-hits/logs/generated.log. | ||||||||||||||||||||||||||||||
| docker_up -profile ${CONFIG_PROFILES}/${PROFILE} -network ${NETWORK_NAME} test-hits | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| # Add the package resources. | ||||||||||||||||||||||||||||||
| add_package -profile ${CONFIG_PROFILES}/${PROFILE} | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| # Add the data stream. | ||||||||||||||||||||||||||||||
| # | ||||||||||||||||||||||||||||||
| # The configuration for the test is described in test_config.yaml below. | ||||||||||||||||||||||||||||||
| add_package_policy -profile ${CONFIG_PROFILES}/${PROFILE} test_config.yaml DATA_STREAM_NAME | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| # Start the service. | ||||||||||||||||||||||||||||||
| docker_signal test-hits SIGHUP | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| # Wait for the service to exit. | ||||||||||||||||||||||||||||||
| docker_wait_exit -timeout 5m test-hits | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| # Check that we can see our policy. | ||||||||||||||||||||||||||||||
| get_policy -profile ${CONFIG_PROFILES}/${PROFILE} -timeout 1m ${DATA_STREAM_NAME} | ||||||||||||||||||||||||||||||
| cp stdout got_policy.json | ||||||||||||||||||||||||||||||
jsoriano marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||||||||||||||||||||||||||
| exec jq '.name=="'${DATA_STREAM_NAME}'"' got_policy.json | ||||||||||||||||||||||||||||||
jsoriano marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||||||||||||||||||||||||||
| stdout true | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| # Take down the service and check logs for our message. | ||||||||||||||||||||||||||||||
| docker_down test-hits | ||||||||||||||||||||||||||||||
| ! stderr . | ||||||||||||||||||||||||||||||
| stdout '"total_lines":10' | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| # Get documents from the data stream. | ||||||||||||||||||||||||||||||
| get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 10 -timeout 5m ${DATA_STREAM_NAME} | ||||||||||||||||||||||||||||||
| cp stdout got_docs.json | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| # Remove the data stream. | ||||||||||||||||||||||||||||||
| remove_package_policy -profile ${CONFIG_PROFILES}/${PROFILE} ${DATA_STREAM_NAME} | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| # Uninstall the agent. | ||||||||||||||||||||||||||||||
| uninstall_agent -profile ${CONFIG_PROFILES}/${PROFILE} -timeout 1m | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| # Remove the package resources. | ||||||||||||||||||||||||||||||
| remove_package -profile ${CONFIG_PROFILES}/${PROFILE} | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| -- test-hits/docker-compose.yml -- | ||||||||||||||||||||||||||||||
| version: '2.3' | ||||||||||||||||||||||||||||||
| services: | ||||||||||||||||||||||||||||||
| test-hits: | ||||||||||||||||||||||||||||||
| image: docker.elastic.co/observability/stream:v0.20.0 | ||||||||||||||||||||||||||||||
| volumes: | ||||||||||||||||||||||||||||||
| - ./logs:/logs:ro | ||||||||||||||||||||||||||||||
| command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9999 -p=tcp /logs/generated.log | ||||||||||||||||||||||||||||||
| -- test-hits/logs/generated.log -- | ||||||||||||||||||||||||||||||
| ntpd[1001]: kernel time sync enabled utl | ||||||||||||||||||||||||||||||
| restorecond: : Reset file context quasiarc: liqua | ||||||||||||||||||||||||||||||
| auditd[5699]: Audit daemon rotating log files | ||||||||||||||||||||||||||||||
| anacron[5066]: Normal exit ehend | ||||||||||||||||||||||||||||||
| restorecond: : Reset file context vol: luptat | ||||||||||||||||||||||||||||||
| heartbeat: : <<eumiu.medium> Processing command: accept | ||||||||||||||||||||||||||||||
| restorecond: : Reset file context nci: ofdeFin | ||||||||||||||||||||||||||||||
| auditd[6668]: Audit daemon rotating log files | ||||||||||||||||||||||||||||||
| anacron[1613]: Normal exit mvolu | ||||||||||||||||||||||||||||||
| ntpd[2959]: ntpd gelit-r tatno | ||||||||||||||||||||||||||||||
| -- test_config.yaml -- | ||||||||||||||||||||||||||||||
| input: tcp | ||||||||||||||||||||||||||||||
| vars: ~ | ||||||||||||||||||||||||||||||
| data_stream: | ||||||||||||||||||||||||||||||
| vars: | ||||||||||||||||||||||||||||||
| tcp_host: 0.0.0.0 | ||||||||||||||||||||||||||||||
| tcp_port: 9999 | ||||||||||||||||||||||||||||||
| ``` | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| Other complete examples can be found in the [with_script test package](https://github.com/elastic/elastic-package/blob/main/test/packages/other/with_script/data_stream/first/_dev/test/scripts). | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| ## Running script tests | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| The `elastic-package test script` command has the following sub-command-specific | ||||||||||||||||||||||||||||||
| flags: | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| - `--continue`: continue running the script if an error occurs | ||||||||||||||||||||||||||||||
| - `--data-streams`: comma-separated data streams to test | ||||||||||||||||||||||||||||||
| - `--external-stack`: use external stack for script tests (default true) | ||||||||||||||||||||||||||||||
| - `--run`: run only tests matching the regular expression | ||||||||||||||||||||||||||||||
| - `--scripts`: path to directory containing test scripts (advanced use only) | ||||||||||||||||||||||||||||||
| - `--update`: update archive file if a cmp fails | ||||||||||||||||||||||||||||||
jsoriano marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||||||||||||||||||||||||||
| - `--verbose-scripts`: verbose script test output (show all script logging) | ||||||||||||||||||||||||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would it be possible to use the general There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please no; with There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Then I would rather try to fix this than adding a subcommand-specific mechanism for logging verbosity. We should probably start using named loggers, and some global selector flag. cc @mrodm There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
At least, the current logger set in elastic-package/internal/logger/logger.go Lines 52 to 65 in 457d321
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Verbosity is wired deeply throughout elastic-package, this is not under the control of the new addition here. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
The issue here is that using a scalar for verbosity ignores that the information being emitted by the script tests is orthogonal to the text that is emitted by the global logger. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Created issue to improve this (#3058) so it is possible to do something like: |
||||||||||||||||||||||||||||||
| - `--work`: print temporary work directory and do not remove when done | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| ## Limitations | ||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||
| While the testscript package allows reference to paths outside the configuration | ||||||||||||||||||||||||||||||
| root and the package's root, the backing `elastic-package` infrastructure does | ||||||||||||||||||||||||||||||
| not, so it is advised that tests only refer to paths within the `$WORK` and | ||||||||||||||||||||||||||||||
| `$PKG_ROOT` directories. | ||||||||||||||||||||||||||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This does not appear to run in the CI, but it's entirely unclear how to achieve that.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
In order to be executed in the CI, it needs to be updated this shell script that generates dynamically all the Buildkite steps to be executed:
https://github.com/elastic/elastic-package/blob/d5f73ab15af1dcf3547ed789f8ad4631257f4197/.buildkite/pipeline.trigger.integration.tests.sh
Depending on how it is required to launch these steps, all packages in one step or each package in its own CI step, it would be needed to do different modifications:
with-kindorotherCI step (Buildkite link)elastic-package/.buildkite/pipeline.trigger.integration.tests.sh
Lines 42 to 47 in d5f73ab
parallelfolder:elastic-package/.buildkite/pipeline.trigger.integration.tests.sh
Lines 117 to 138 in d5f73ab