add cli meta tool pkg and build job (#384)

* add cli meta tool pkg and build job

azure-cli-diff-tool/HISTORY.rst

@@ -0,0 +1,8 @@
+.. :changelog:
+
+Release History
+===============
+
+0.0.1
+++++++
+* Initial release

azure-cli-diff-tool/README.rst

@@ -0,0 +1,84 @@
+Microsoft Azure CLI Diff Tools (azureCliDiffTool)
+=====================================
+
+The ``azureCliDiffTool`` is designed to aid azure-cli users in diffing metadata files to see its updates through historical versions for Azure CLI command modules and extensions.
+
+Setting up your environment
++++++++++++++++++++++++++++++++++++++++
+
+1. Install Python 3.6+ from http://python.org. Please note that the version of Python that comes preinstalled on OSX is 2.7.
+
+3. Create a new virtual environment for Python in the root of your clone. You can do this by running:
+
+    Python 3.6+ (all platforms):
+
+    ::
+
+        python -m venv env
+
+    or:
+
+    ::
+
+        python3 -m venv env
+
+
+4. Activate the env virtual environment by running:
+
+    Windows CMD.exe:
+
+    ::
+
+        env\scripts\activate.bat
+
+    Windows Powershell:
+
+    ::
+
+        env\scripts\activate.ps1
+
+
+    OSX/Linux (bash):
+
+    ::
+
+        source env/bin/activate
+
+5. Install ``azureCliDiffTool`` by running:
+
+    ::
+
+        pip install azure-cli-diff-tool
+
+Reporting issues and feedback
++++++++++++++++++++++++++++++
+
+If you encounter any bugs with the tool please file an issue in the `Issues <https://github.com/Azure/azure-cli-dev-tools/issues>`__ section of our GitHub repo.
+
+Contribute Code
++++++++++++++++
+
+This project has adopted the `Microsoft Open Source Code of Conduct <https://opensource.microsoft.com/codeofconduct/>`__.
+
+For more information see the `Code of Conduct FAQ <https://opensource.microsoft.com/codeofconduct/faq/>`__ or contact `[email protected] <mailto:[email protected]>`__ with any additional questions or comments.
+
+If you would like to become an active contributor to this project please
+follow the instructions provided in `Microsoft Azure Projects Contribution Guidelines <http://azure.github.io/guidelines.html>`__.
+
+License
++++++++
+
+::
+
+    Azure CLI Diff Tools (azureCliDiffTool)
+
+    Copyright (c) Microsoft Corporation
+    All rights reserved.
+
+    MIT License
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the ""Software""), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.::

azure-cli-diff-tool/azureCliDiffTool/__init__.py

@@ -0,0 +1,135 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+
+# pylint: disable=too-many-lines
+# pylint: disable=too-many-statements
+import os
+import json
+import time
+from enum import Enum
+import logging
+from deepdiff import DeepDiff
+from .meta_change_detect import MetaChangeDetect
+from .utils import get_blob_config, load_blob_config_file, get_target_version_modules, get_target_version_module, \
+    extract_module_name_from_meta_file, export_meta_changes_to_csv, export_meta_changes_to_json, \
+    export_meta_changes_to_dict
+
+__VERSION__ = '0.0.1'
+
+logger = logging.getLogger(__name__)
+
+
+class DiffExportFormat(Enum):
+    DICT = "dict"
+    TEXT = "text"
+    TREE = "tree"
+
+
+def diff_export_format_choices():
+    return [form.value for form in DiffExportFormat]
+
+
+def check_meta_tool_compatibility(meta_version):
+    if not meta_version:
+        return False
+    meta_version_vec = meta_version.split(".")
+    tool_version_vec = __VERSION__.split(".")
+    version_outdated = False
+    for ind, v in enumerate(meta_version_vec):
+        if v.isdigit() and tool_version_vec[ind].isdigit() and int(v) > int(tool_version_vec[ind]):
+            version_outdated = True
+    return version_outdated
+
+
+def meta_diff(base_meta_file, diff_meta_file, only_break=False, output_type="text", output_file=None):
+    if not os.path.exists(base_meta_file):
+        raise Exception("base meta file needed")
+    if not os.path.exists(diff_meta_file):
+        raise Exception("diff meta file needed")
+
+    with open(base_meta_file, "r") as g:
+        command_tree_before = json.load(g)
+    with open(diff_meta_file, "r") as g:
+        command_tree_after = json.load(g)
+    if command_tree_before.get("compat_version", None) \
+            and check_meta_tool_compatibility(command_tree_before["compat_version"]):
+        raise Exception("Please update your azure cli diff tool")
+    if command_tree_after.get("compat_version", None) \
+            and check_meta_tool_compatibility(command_tree_after["compat_version"]):
+        raise Exception("Please update your azure cli diff tool")
+
+    diff = DeepDiff(command_tree_before, command_tree_after)
+    if not diff:
+        print(f"No meta diffs from {diff_meta_file} to {base_meta_file}")
+        return export_meta_changes_to_json(None, output_file)
+    else:
+        detected_changes = MetaChangeDetect(diff, command_tree_before, command_tree_after)
+        detected_changes.check_deep_diffs()
+        result = detected_changes.export_meta_changes(only_break, output_type)
+        return export_meta_changes_to_json(result, output_file)
+
+
+def version_diff(base_version, diff_version, only_break=False, version_diff_file=None, use_cache=False,
+                 output_type="dict", target_module=None):
+    config = load_blob_config_file()
+    blob_url, path_prefix, index_file = get_blob_config(config)
+    download_base_start = time.time()
+    if target_module:
+        base_version_module_list = get_target_version_module(blob_url, path_prefix, base_version,
+                                                             target_module, use_cache)
+    else:
+        base_version_module_list = get_target_version_modules(blob_url, path_prefix, index_file, base_version, use_cache)
+    download_base_end = time.time()
+    print("base version {} meta files download using {} sec".format(base_version,
+                                                                          download_base_end - download_base_start))
+    if target_module:
+        get_target_version_module(blob_url, path_prefix, diff_version,
+                                  target_module, use_cache)
+    else:
+        get_target_version_modules(blob_url, path_prefix, index_file, diff_version, use_cache)
+    download_target_end = time.time()
+    print("diff version {} meta files download using {} sec".format(diff_version,
+                                                                          download_target_end - download_base_end))
+    version_diffs = []
+    for _, base_meta_file_full_path, base_meta_file in base_version_module_list:
+        module_name = extract_module_name_from_meta_file(base_meta_file)
+        if not module_name:
+            continue
+        if target_module and module_name != target_module:
+            continue
+        diff_meta_file_full_path = os.path.join(os.getcwd(), path_prefix + diff_version, base_meta_file)
+        if not os.path.exists(diff_meta_file_full_path):
+            print(f"Module {module_name} removed for {diff_version}")
+            continue
+        with open(base_meta_file_full_path, "r") as g:
+            command_tree_before = json.load(g)
+        with open(diff_meta_file_full_path, "r") as g:
+            command_tree_after = json.load(g)
+
+        if command_tree_before.get("compat_version", None) \
+                and check_meta_tool_compatibility(command_tree_before["compat_version"]):
+            raise Exception("Please update your azure cli diff tool")
+        if command_tree_after.get("compat_version", None) \
+                and check_meta_tool_compatibility(command_tree_after["compat_version"]):
+            raise Exception("Please update your azure cli diff tool")
+
+        diff = DeepDiff(command_tree_before, command_tree_after)
+        if not diff:
+            print(f"No meta diffs from version: {diff_version}/{base_meta_file} for module: {module_name}")
+            continue
+        detected_changes = MetaChangeDetect(diff, command_tree_before, command_tree_after)
+        detected_changes.check_deep_diffs()
+        diff_objs = detected_changes.export_meta_changes(only_break, "dict")
+        mod_obj = {"module": module_name}
+        for obj in diff_objs:
+            obj.update(mod_obj)
+            version_diffs.append(obj)
+    meta_change_end = time.time()
+    print("meta file diffs using {} sec".format(meta_change_end - download_target_end))
+    if output_type == "dict":
+        return export_meta_changes_to_dict(version_diffs, version_diff_file)
+    return export_meta_changes_to_csv(version_diffs, version_diff_file)

azure-cli-diff-tool/azureCliDiffTool/_const.py

@@ -0,0 +1,67 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+import os
+BLOB_SETTING_CONFIG_FILE = "./data/blob_config.ini"
+script_directory = os.path.dirname(os.path.realpath(__file__))
+CONFIG_FILE_PATH = f"{script_directory}/{BLOB_SETTING_CONFIG_FILE}"
+
+META_CHANDE_WHITELIST_FILE = "./data/meta_change_whitelist.txt"
+META_CHANDE_WHITELIST_FILE_PATH = f"{script_directory}/{META_CHANDE_WHITELIST_FILE}"
+
+DOWNLOAD_THREADS = 30
+
+BREAKING_CHANE_RULE_LINK_URL_PREFIX = "https://github.com/Azure/azure-cli/blob/dev/doc/breaking_change_rules/"
+BREAKING_CHANE_RULE_LINK_URL_SUFFIX = ".md"
+
+CMD_PROPERTY_REMOVE_BREAK_LIST = []
+CMD_PROPERTY_ADD_BREAK_LIST = ["confirmation"]
+CMD_PROPERTY_UPDATE_BREAK_LIST = []
+CMD_PROPERTY_IGNORED_LIST = ["is_aaz", "supports_no_wait"]
+
+PARA_PROPERTY_REMOVE_BREAK_LIST = ["options", "id_part", "nargs"]
+PARA_PROPERTY_ADD_BREAK_LIST = ["required", "choices", "type"]
+PARA_PROPERTY_UPDATE_BREAK_LIST = ["default", "aaz_default", "type"]
+PARA_NAME_IGNORED_LIST = ["force_string"]
+PARA_PROPERTY_IGNORED_LIST = []
+PARA_VALUE_IGNORED_LIST = ["generic_update_set", "generic_update_add", "generic_update_remove",
+                           "generic_update_force_string"]
+
+EXPORTED_CSV_META_HEADER = ["module", "cmd_name", "rule_id", "rule_name", "is_break", "rule_link_url",
+                            "rule_message", "suggest_message"]
+
+CHANGE_RULE_MESSAGE_MAPPING = {
+    "1000": "default Message",
+    "1001": "cmd `{0}` added",
+    "1002": "cmd `{0}` removed",
+    "1003": "cmd `{0}` added property `{1}`",
+    "1004": "cmd `{0}` removed property `{1}`",
+    "1005": "cmd `{0}` updated property `{1}` from `{2}` to `{3}`",
+    "1006": "cmd `{0}` added parameter `{1}`",
+    "1007": "cmd `{0}` removed parameter `{1}`",
+    "1008": "cmd `{0}` update parameter `{1}`: added property `{2}`",
+    "1009": "cmd `{0}` update parameter `{1}`: removed property `{2}`",
+    "1010": "cmd `{0}` update parameter `{1}`: updated property `{2}` from `{3}` to `{4}`",
+    "1011": "sub group `{0}` added",
+    "1012": "sub group `{0}` removed",
+}
+
+CHANGE_SUGGEST_MESSAGE_MAPPING = {
+    "1000": "default Message",
+    "1001": "please confirm cmd `{0}` added",
+    "1002": "please confirm cmd `{0}` removed",
+    "1003": "please remove property `{0}` for cmd `{1}`",
+    "1004": "please add back property `{0}` for cmd `{1}`",
+    "1005": "please change property `{0}` from `{1}` to `{2}` for cmd `{3}`",
+    "1006": "please remove parameter `{0}` for cmd `{1}`",
+    "1007": "please add back parameter `{0}` for cmd `{1}`",
+    "1008": "please remove property `{0}` for parameter `{1}` for cmd `{2}`",
+    "1009": "please add back property `{0}` for parameter {1}` for cmd `{2}`",
+    "1010": "please change property `{0}` from `{1}` to `{2}` for parameter `{3}` of cmd `{4}`",
+    "1011": "please confirm sub group `{0}` added",
+    "1012": "please confirm sub group `{0}` removed",
+}
+

azure-cli-diff-tool/azureCliDiffTool/data/blob_config.ini

@@ -0,0 +1,4 @@
+[BLOB]
+primary_endpoint=https://azcmdchangemgmt.blob.core.windows.net/cmd-metadata-per-version
+metadata_path_prefix=azure-cli-
+metadata_module_index_file=index.txt

azure-cli-diff-tool/azureCliDiffTool/data/meta_change_whitelist.txt

@@ -0,0 +1,9 @@
+1010	ams asset get-sas-urls	expiry_time	default
+1010	mysql flexible-server create	administrator_login	default
+1010	mysql flexible-server restore	restore_point_in_time	default
+1010	mysql flexible-server import create	administrator_login	default
+1010	postgres flexible-server create	administrator_login	default
+1010	postgres flexible-server restore	restore_point_in_time	default
+1010	mysql server create	administrator_login	default
+1010	postgres server create	administrator_login	default
+1010	mariadb server create	administrator_login	default