Add support for API version 2024-11-15-preview for Language AnalyzeDo…

…cuments (#31088) * Add Changes for Language - Documents API * Another update * update * fix * add examples * typespec changes * add excludeEntities parameter * tsv * small change * one more update * Update for query * run tsv * typo * remove v2023_11_15_preview --------- Co-authored-by: Bidisha Chakraborty <[email protected]>
Azure · Nov 8, 2024 · 956f8dd · 956f8dd
1 parent 033eb83
commit 956f8dd
Show file tree

Hide file tree

Showing 12 changed files with 4,631 additions and 33 deletions.
diff --git a/...nts/examples/2024-11-15-preview/SuccessfulAnalyzeDocumentsJobStatusTaskStatusRequest.json b/...nts/examples/2024-11-15-preview/SuccessfulAnalyzeDocumentsJobStatusTaskStatusRequest.json
@@ -0,0 +1,61 @@
+{
+  "operationId": "AnalyzeDocumentsJobStatus",
+  "title": "SuccessfulAnalyzeDocumentsJobStatusTaskStatusRequest",
+  "parameters": {
+    "Ocp-Apim-Subscription-Key": "{API key}",
+    "api-version": "2024-11-15-preview",
+    "Endpoint": "{Endpoint}",
+    "jobId": "c0f2a446-05d9-48fc-ba8f-3ef4af8d0b18"
+  },
+  "responses": {
+    "200": {
+      "headers": {},
+      "body": {
+        "createdDateTime": "2023-12-30T15:00:45Z",
+        "displayName": "Extracting Location & US Region",
+        "expirationDateTime": "2020-10-03T15:01:03Z",
+        "jobId": "c0f2a446-05d9-48fc-ba8f-3ef4af8d0b18",
+        "lastUpdatedDateTime": "2023-12-30T15:01:03Z",
+        "status": "succeeded",
+        "tasks": {
+          "completed": 2,
+          "failed": 0,
+          "inProgress": 0,
+          "total": 2,
+          "items": [
+            {
+              "kind": "PiiEntityRecognitionLROResults",
+              "taskName": "PII Redaction Task 1",
+              "lastUpdateDateTime": "2020-10-01T15:01:03Z",
+              "status": "succeeded",
+              "results": {
+                "documents": [
+                  {
+                    "source": {
+                      "location": "https://myblob.blob.core.windows.net/Container/document.txt",
+                      "kind": "AzureBlob"
+                    },
+                    "target": [
+                      {
+                        "location": "https://myblob.blob.core.windows.net/Container/0af5def0-5855-41b2-9e75-0a559fb6f545/PiiEntityRecognition/1/document.txt",
+                        "kind": "AzureBlob"
+                      },
+                      {
+                        "location": "https://myblob.blob.core.windows.net/Container/0af5def0-5855-41b2-9e75-0a559fb6f545/PiiEntityRecognition/1/document.json",
+                        "kind": "AzureBlob"
+                      }
+                    ],
+                    "id": "1",
+                    "warnings": []
+                  }
+                ],
+                "errors": [],
+                "modelVersion": "2023-09-01"
+              }
+            }
+          ]
+        }
+      }
+    }
+  }
+}
diff --git a/...yzeDocuments/examples/2024-11-15-preview/SuccessfulAnalyzeDocumentsJobsCancelRequest.json b/...yzeDocuments/examples/2024-11-15-preview/SuccessfulAnalyzeDocumentsJobsCancelRequest.json
@@ -0,0 +1,17 @@
+{
+  "operationId": "AnalyzeDocumentsCancelJob",
+  "title": "SuccessfulAnalyzeDocumentsJobsCancelRequest",
+  "parameters": {
+    "Ocp-Apim-Subscription-Key": "{API key}",
+    "api-version": "2024-11-15-preview",
+    "Endpoint": "{Endpoint}",
+    "jobId": "c0f2a446-05d9-48fc-ba8f-3ef4af8d0b18"
+  },
+  "responses": {
+    "202": {
+      "headers": {
+        "Operation-Location": "{Endpoint}/language/analyze-documents/jobs/{jobId}?api-version=2024-11-15-preview"
+      }
+    }
+  }
+}
diff --git a/...ocuments/examples/2024-11-15-preview/SuccessfulAnalyzeDocumentsJobsTaskSubmitRequest.json b/...ocuments/examples/2024-11-15-preview/SuccessfulAnalyzeDocumentsJobsTaskSubmitRequest.json
@@ -0,0 +1,51 @@
+{
+  "operationId": "AnalyzeDocumentsSubmitJob",
+  "title": "SuccessfulAnalyzeDocumentsJobsTaskSubmitRequest",
+  "parameters": {
+    "Ocp-Apim-Subscription-Key": "{API key}",
+    "api-version": "2024-11-15-preview",
+    "Endpoint": "{Endpoint}",
+    "jobId": "{Job ID}",
+    "body": {
+      "displayName": "Redact PII from documents example",
+      "analysisInput": {
+        "documents": [
+          {
+            "id": "1",
+            "language": "en",
+            "source": {
+              "location": "https://myblob.blob.core.windows.net/Container/document.txt",
+              "kind": "AzureBlob"
+            },
+            "target": {
+              "location": "https://myblob.blob.core.windows.net/Container",
+              "kind": "AzureBlob"
+            }
+          }
+        ]
+      },
+      "tasks": [
+        {
+          "kind": "PiiEntityRecognition",
+          "taskName": "Redact PII Task 1",
+          "parameters": {
+            "redactionPolicy": {
+              "policyKind": "entityMask"
+            },
+            "piiCategories": [
+              "Person",
+              "Organization"
+            ]
+          }
+        }
+      ]
+    }
+  },
+  "responses": {
+    "202": {
+      "headers": {
+        "Operation-Location": "{endpoint}/language/analyze-documents/jobs/{jobId}?api-version=2024-11-15-preview"
+      }
+    }
+  }
+}
diff --git a/specification/cognitiveservices/Language.AnalyzeDocuments/main.tsp b/specification/cognitiveservices/Language.AnalyzeDocuments/main.tsp
@@ -39,9 +39,9 @@ namespace Language.AnalyzeDocuments;
 
 /** Analyze Documents Service Versions */
 enum Versions {
-  /** Version 2023-11-15-preview */
+  /** Version 2024-11-15-preview */
   @useDependency(Azure.Core.Versions.v1_0_Preview_2)
-  v2023_11_15_preview: "2023-11-15-preview",
+  v2024_11_15_preview: "2024-11-15-preview",
 }
 
 alias ServiceTraits = NoRepeatableRequests &

diff --git a/specification/cognitiveservices/Language.AnalyzeDocuments/models/common.models.tsp b/specification/cognitiveservices/Language.AnalyzeDocuments/models/common.models.tsp
@@ -117,7 +117,6 @@ model DetectedLanguage {
 }
 
 /** The script of the text. */
-@added(Versions.v2023_11_15_preview)
 union ScriptKind {
   string,
 
@@ -210,7 +209,6 @@ union ScriptKind {
 }
 
 /** Identifies the script of the input document. Maps to the ISO 15924 standard script code. */
-@added(Versions.v2023_11_15_preview)
 union ScriptCode {
   string,
 

diff --git a/specification/cognitiveservices/Language.AnalyzeDocuments/models/pii.entity.recognition.tsp b/specification/cognitiveservices/Language.AnalyzeDocuments/models/pii.entity.recognition.tsp
@@ -21,8 +21,13 @@ model PiiTaskParameters is PreBuiltTaskParameters {
   /** Enumeration of PII categories to be excluded in the response. */
   excludePiiCategories?: Array<PiiCategoriesExclude>;
 
-  /** Character to be used for redaction. Default in '*'. */
-  redactionCharacter?: redactionCharacter = redactionCharacter.asterisk;
+  /** Redaction policy to be used when redacting from documents */
+  @added(Versions.v2024_11_15_preview)
+  redactionPolicy?: BaseRedactionPolicy;
+
+  /** Boolean to indicate if entities should be excluded from the response json stored in the customer's storage. */
+  @added(Versions.v2024_11_15_preview)
+  excludeExtractionData?: boolean;
 }
 
 /** Contains the analyze text PIIEntityRecognition LRO task. */
@@ -53,6 +58,48 @@ union PiiDomain {
   none: "none",
 }
 
+/** The abstract base class for RedactionPolicy. */
+@discriminator("policyKind")
+model BaseRedactionPolicy {
+  /** The entity RedactionPolicy object kind. */
+  policyKind: RedactionPolicyKind;
+}
+
+/** Represents the policy of redacting with a redaction character */
+model CharacterMaskPolicy extends BaseRedactionPolicy {
+  /** The entity RedactionPolicy object kind. */
+  policyKind: RedactionPolicyKind.characterMask;
+
+  /** Optional parameter to use a Custom Character to be used for redaction in PII responses. Default character will be * as before. We allow specific ascii characters for redaction. */
+  redactionCharacter?: redactionCharacter = redactionCharacter.asterisk;
+}
+
+/** Represents the policy of not redacting found PII. */
+model NoMaskPolicy extends BaseRedactionPolicy {
+  /** The entity RedactionPolicy object kind. */
+  policyKind: RedactionPolicyKind.noMask;
+}
+
+/** Represents the policy of redacting PII with the entity type. */
+model EntityMaskPolicy extends BaseRedactionPolicy {
+  /** The entity OverlapPolicy object kind. */
+  policyKind: RedactionPolicyKind.entityMask;
+}
+
+/** Kinds of redaction policies supported. */
+union RedactionPolicyKind {
+  string,
+
+  /** Do not redact detected entities. */
+  noMask: "noMask",
+
+  /** React detected entities with redaction character. */
+  characterMask: "characterMask",
+
+  /** Redact detected entities with entity type. */
+  entityMask: "entityMask",
+}
+
 /** Contains the PII results. */
 #suppress "@azure-tools/typespec-azure-core/casing-style"
 model PiiEntitiesDocumentResult is DocumentResult {
@@ -74,20 +121,29 @@ model Entity {
   /** Entity text as appears in the request. */
   text: string;
 
-  /** Entity type. */
-  category: string;
-
-  /** (Optional) Entity sub type. */
-  subcategory?: string;
-
-  /** Start position for the entity text. Use of different 'stringIndexType' values can affect the offset returned. */
-  offset: int32;
-
   /** Length for the entity text. Use of different 'stringIndexType' values can affect the length returned. */
   length: int32;
 
   /** Confidence score between 0 and 1 of the extracted entity. */
   confidenceScore: float64;
+
+  /** Most granular type of the entity detected */
+  type: string;
+
+  /** An identifier for mappping the extracted entity with the redacted data in the original document. */
+  entityId: string;
+
+  /** List of entity tags. Tags are to express some similarities/affinity between entities. */
+  tags: Array<EntityTag>;
+}
+
+/** Entity tag expresses any affinity or association the entity has. Example if it part of a broarding grouping like a Numeric or Location entity, etc. */
+model EntityTag {
+  /** Name of the tag. Entity Tag names will be unique globally. */
+  name: string;
+
+  /** Detection score between 0 and 1 of the entity tag. */
+  confidenceScore?: float64;
 }
 
 /** (Optional) describes the PII categories to return */