Skip to content

Commit 6dd9d07

Browse files
CoromTCoromT
committed
Bring Your Own Data marketplace service template derived from the FBT service
1 parent 4177ddc commit 6dd9d07

39 files changed

+5019
-0
lines changed

‎.gitattributes

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
###############################################################################
2+
# Set default behavior to automatically normalize line endings.
3+
###############################################################################
4+
* text=auto
5+
6+
###############################################################################
7+
# Set default behavior for command prompt diff.
8+
#
9+
# This is need for earlier builds of msysgit that does not have it on by
10+
# default for csharp files.
11+
# Note: This is only used by command line
12+
###############################################################################
13+
#*.cs diff=csharp
14+
15+
###############################################################################
16+
# Set the merge driver for project and solution files
17+
#
18+
# Merging from the command prompt will add diff markers to the files if there
19+
# are conflicts (Merging from VS is not affected by the settings below, in VS
20+
# the diff markers are never inserted). Diff markers may cause the following
21+
# file extensions to fail to load in VS. An alternative would be to treat
22+
# these files as binary and thus will always conflict and require user
23+
# intervention with every merge. To do so, just uncomment the entries below
24+
###############################################################################
25+
#*.sln merge=binary
26+
#*.csproj merge=binary
27+
#*.vbproj merge=binary
28+
#*.vcxproj merge=binary
29+
#*.vcproj merge=binary
30+
#*.dbproj merge=binary
31+
#*.fsproj merge=binary
32+
#*.lsproj merge=binary
33+
#*.wixproj merge=binary
34+
#*.modelproj merge=binary
35+
#*.sqlproj merge=binary
36+
#*.wwaproj merge=binary
37+
38+
###############################################################################
39+
# behavior for image files
40+
#
41+
# image files are treated as binary by default.
42+
###############################################################################
43+
#*.jpg binary
44+
#*.png binary
45+
#*.gif binary
46+
47+
###############################################################################
48+
# diff behavior for common document formats
49+
#
50+
# Convert binary document formats to text before diffing them. This feature
51+
# is only available from the command line. Turn it on by uncommenting the
52+
# entries below.
53+
###############################################################################
54+
#*.doc diff=astextplain
55+
#*.DOC diff=astextplain
56+
#*.docx diff=astextplain
57+
#*.DOCX diff=astextplain
58+
#*.dot diff=astextplain
59+
#*.DOT diff=astextplain
60+
#*.pdf diff=astextplain
61+
#*.PDF diff=astextplain
62+
#*.rtf diff=astextplain
63+
#*.RTF diff=astextplain

‎Apps/ByodService/App_Start/WebApiConfig.cs

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
using MarketBasket.Web.Controllers;
2+
using System;
3+
using System.Net.Http;
4+
using System.Web;
5+
using System.Web.Http;
6+
using System.Web.Http.Hosting;
7+
using System.Web.Http.WebHost;
8+
9+
namespace MarketBasket.Web
10+
{
11+
12+
public static class WebApiConfig
13+
{
14+
public static void Register(HttpConfiguration config)
15+
{
16+
// Use a streaming policy for upload
17+
config.Services.Replace(typeof(IHostBufferPolicySelector), new NoBufferPolicySelector());
18+
19+
// Help Controller for UI
20+
config.Routes.MapHttpRoute(
21+
name: "Help",
22+
routeTemplate: "odata/help/{*path}",
23+
defaults: new { controller = "Help" }
24+
);
25+
26+
// Controller for metadata catalog
27+
config.Routes.MapHttpRoute(
28+
name: "Catalog",
29+
routeTemplate: "Metadata",
30+
defaults: new { controller = "Catalog" }
31+
);
32+
33+
// Controller for OAuth
34+
config.Routes.MapHttpRoute(
35+
name: "OAuth",
36+
routeTemplate: "AuthorizeUser",
37+
defaults: new { controller = "OAuth" }
38+
);
39+
40+
// Marketplace Odata API
41+
config.MapOdataActionApi<FrequentlyBoughtTogetherController>("odata");
42+
}
43+
}
44+
45+
46+
public class NoBufferPolicySelector : WebHostBufferPolicySelector
47+
{
48+
public override bool UseBufferedInputStream(object hostContext)
49+
{
50+
51+
var context = hostContext as HttpContextBase;
52+
return context != null && context.Request.HttpMethod == "POST";
53+
}
54+
55+
public override bool UseBufferedOutputStream(HttpResponseMessage response)
56+
{
57+
return base.UseBufferedOutputStream(response);
58+
}
59+
}
60+
61+
62+
}

‎Apps/ByodService/AuthorizeApiKeyAttribute.cs

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Net;
5+
using System.Net.Http;
6+
using System.Net.Http.Headers;
7+
using System.Text;
8+
using System.Web;
9+
using System.Web.Http.Controllers;
10+
using System.Web.Http.Filters;
11+
12+
namespace MarketBasket.Web
13+
{
14+
15+
// Does basic authentication using user provided API keys
16+
public class ApiKeyAuthorizationAttribute : ActionFilterAttribute
17+
{
18+
private string[] apiKeys;
19+
20+
public ApiKeyAuthorizationAttribute(params string[] apiKeys)
21+
{
22+
this.apiKeys = apiKeys;
23+
}
24+
25+
public override void OnActionExecuting(HttpActionContext actionContext)
26+
{
27+
var authHead = actionContext.Request.Headers.Authorization;
28+
if (authHead != null)
29+
{
30+
if (authHead.Scheme.Equals("basic", StringComparison.OrdinalIgnoreCase) && authHead.Parameter != null)
31+
{
32+
var credentials = Encoding.GetEncoding("iso-8859-1").GetString(Convert.FromBase64String(authHead.Parameter));
33+
34+
int separator = credentials.IndexOf(':');
35+
string username = credentials.Substring(0, separator);
36+
string password = credentials.Substring(separator + 1);
37+
38+
if (apiKeys.Contains(password))
39+
{
40+
IUserScopedController controller = actionContext.ControllerContext.Controller as IUserScopedController;
41+
if (controller != null && !string.IsNullOrWhiteSpace(username))
42+
{
43+
controller.UserId = username;
44+
}
45+
return;
46+
}
47+
}
48+
}
49+
50+
actionContext.Response = new HttpResponseMessage(HttpStatusCode.Unauthorized);
51+
actionContext.Response.Headers.Add("WWW-Authenticate", string.Format("Basic realm=\"{0}\"", actionContext.Request.RequestUri.Host));
52+
}
53+
}
54+
55+
public interface IUserScopedController
56+
{
57+
string UserId { get; set; }
58+
}
59+
60+
}

‎Apps/ByodService/AzureMLClient/Client.cs

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Globalization;
4+
using System.Net.Http;
5+
using System.Threading.Tasks;
6+
using AzureMLClient.Contracts;
7+
8+
namespace AzureMLClient
9+
{
10+
public class Client
11+
{
12+
private const string WebServiceUriFormat = "workspaces/{0}/webservices/{1}";
13+
14+
public async Task<IEnumerable<WebService>> GetWebServicesAsync()
15+
{
16+
using (var httpClient = new HttpClient())
17+
{
18+
string uri = string.Format(CultureInfo.InvariantCulture, WebServiceUriFormat, Configurations.WorkspaceId, string.Empty);
19+
httpClient.DefaultRequestHeaders.Authorization = ClientExtensions.GetAuthorizationHeaderAsync(Configurations.WorkspaceAuthToken);
20+
var response = await httpClient.GetAsync(new Uri(Configurations.BaseUri, uri)).ConfigureAwait(false);
21+
return await response.Content.ReadAsAsync<IEnumerable<WebService>>();
22+
}
23+
}
24+
}
25+
}

‎Apps/ByodService/AzureMLClient/ClientExtensions.cs

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Globalization;
4+
using System.Net;
5+
using System.Net.Http;
6+
using System.Net.Http.Formatting;
7+
using System.Net.Http.Headers;
8+
using System.Threading.Tasks;
9+
using AzureMLClient.Contracts;
10+
11+
namespace AzureMLClient
12+
{
13+
public static class ClientExtensions
14+
{
15+
private const string BatchUriFormat = "{0}/jobs/{1}?api-version=2.0";
16+
private const string EndpointsUriFormat = "workspaces/{0}/webservices/{1}/endpoints";
17+
private const string UpdateResourceUriFormat = "https://management.azureml.net/workspaces/{0}/webservices/{1}/endpoints/{2}";
18+
19+
public static async Task<IEnumerable<Endpoint>> GetEndpointsAsync(this WebService webServiece)
20+
{
21+
string uri = string.Format(CultureInfo.InvariantCulture, EndpointsUriFormat, webServiece.WorkspaceId, webServiece.Id) + "?decryptSecrets=true";
22+
using (var httpClient = new HttpClient())
23+
{
24+
httpClient.DefaultRequestHeaders.Authorization = GetAuthorizationHeaderAsync(Configurations.WorkspaceAuthToken);
25+
var response = await httpClient.GetAsync(new Uri(Configurations.BaseUri, uri)).ConfigureAwait(false);
26+
return await response.Content.ReadAsAsync<IEnumerable<Endpoint>>().ConfigureAwait(false);
27+
}
28+
}
29+
30+
31+
public static async Task<BatchStatus> GetBatchStatus(this Endpoint endpoint, string jobId)
32+
{
33+
using (var httpClient = new HttpClient())
34+
{
35+
var uri = string.Format(CultureInfo.InvariantCulture, BatchUriFormat, endpoint.ApiLocation, jobId);
36+
httpClient.DefaultRequestHeaders.Authorization = GetAuthorizationHeaderAsync(endpoint.PrimaryKey);
37+
var response = await httpClient.GetAsync(new Uri(uri)).ConfigureAwait(false);
38+
return await response.Content.ReadAsAsync<BatchStatus>().ConfigureAwait(false);
39+
}
40+
}
41+
42+
public static async Task<string> SubmitBatch(this Endpoint endpoint, BatchRequest batchRequest)
43+
{
44+
using (var httpClient = new HttpClient())
45+
{
46+
string uri = endpoint.ApiLocation + "/jobs";
47+
httpClient.DefaultRequestHeaders.Authorization = GetAuthorizationHeaderAsync(endpoint.PrimaryKey);
48+
var response = await httpClient.PostAsync(new Uri(uri), new ObjectContent(typeof(BatchRequest), batchRequest, new JsonMediaTypeFormatter())).ConfigureAwait(false);
49+
return await response.Content.ReadAsAsync<string>().ConfigureAwait(false);
50+
}
51+
}
52+
53+
public static async Task<bool> UpdateResource(this Endpoint endpoint, ResourceLocations resources)
54+
{
55+
using (var httpClient = new HttpClient())
56+
{
57+
var uri = string.Format(CultureInfo.InvariantCulture, UpdateResourceUriFormat, endpoint.WorkspaceId, endpoint.WebServiceId, endpoint.Name);
58+
httpClient.DefaultRequestHeaders.Authorization = GetAuthorizationHeaderAsync(endpoint.PrimaryKey);
59+
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
60+
var request = new HttpRequestMessage(new HttpMethod("PATCH"), uri);
61+
request.Content = new ObjectContent(typeof(ResourceLocations), resources, new JsonMediaTypeFormatter());
62+
var response = await httpClient.SendAsync(request).ConfigureAwait(false);
63+
if (response.IsSuccessStatusCode)
64+
{
65+
return true;
66+
}
67+
return false;
68+
}
69+
}
70+
71+
public static async Task<Endpoint> CreateEndpoint(this WebService webservice, string name, EndpointCreate create, string workspaceAuthToken)
72+
{
73+
using (var httpClient = new HttpClient())
74+
{
75+
var uri = string.Format(CultureInfo.InvariantCulture, UpdateResourceUriFormat, webservice.WorkspaceId, webservice.Id, name);
76+
httpClient.DefaultRequestHeaders.Authorization = GetAuthorizationHeaderAsync(workspaceAuthToken);
77+
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
78+
var request = new HttpRequestMessage(new HttpMethod("PUT"), uri);
79+
request.Content = new ObjectContent(typeof(EndpointCreate), create, new JsonMediaTypeFormatter());
80+
var response = await httpClient.SendAsync(request).ConfigureAwait(false);
81+
if (response.IsSuccessStatusCode)
82+
{
83+
return await response.Content.ReadAsAsync<Endpoint>().ConfigureAwait(false);
84+
}
85+
return null;
86+
}
87+
}
88+
89+
public static bool Completed(this BatchStatus status)
90+
{
91+
return status.StatusCode == BatchScoreStatusCode.Cancelled ||
92+
status.StatusCode == BatchScoreStatusCode.Failed ||
93+
status.StatusCode == BatchScoreStatusCode.Finished;
94+
}
95+
96+
public static AuthenticationHeaderValue GetAuthorizationHeaderAsync(string key)
97+
{
98+
return new AuthenticationHeaderValue("Bearer", key);
99+
}
100+
101+
public static string ToUri(this AzureBlobDataReference blob)
102+
{
103+
return blob.BaseLocation + blob.RelativeLocation + blob.SasBlobToken;
104+
}
105+
}
106+
}

‎Apps/ByodService/AzureMLClient/Configurations.cs

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
using System;
2+
3+
namespace AzureMLClient
4+
{
5+
public class Configurations
6+
{
7+
public static readonly Uri BaseUri = new Uri("https://studio.azureml.net/");
8+
9+
// obtain this from url in the azure ml studio
10+
public const string WorkspaceId = "9dbdce0846e64a5f9c925116e0cb6388";
11+
12+
// obtain this from Setting tab in ML Studio
13+
public const string WorkspaceAuthToken = "74ec36c8efec4178b9c868a5df8f9926";
14+
15+
// any storage account you wish to use
16+
public const string AzureStorageConnectionString = "DefaultEndpointsProtocol=https;AccountName=micmancloudmltest2;AccountKey=hENx5QfYRh1kKUHtBPddXiagLrsKSRaB4aK4Eep0Fl3lR00utQ87VXghVBAGd/iyuDVEH8/YHUQfJ4yswg8eeA==";
17+
public const string AzureStorageContainerName = "bes";
18+
19+
// (optional) pre-uploaded blob for retraining - for demo purpose only
20+
public const string AzureStoragePreUploadedBlobName = "Iris Two Class Data.arff";
21+
22+
// you can find the names below in the batch help page of the published web services.
23+
// the retaining trained model output port name, find this in your training experiment, or your batch help page under "Sample Response Payload, if job is finished" section
24+
public const string TrainedModelOutputPortName = "trained model";
25+
// the retaining evaluation output port name, find this in your training experiment, or your batch help page under "Sample Response Payload, if job is finished" section
26+
// note, this is optional and you may comment this out together with the code that attempts to read this
27+
public const string EvaluationOutputPortName = "evaluation result";
28+
29+
// the model input name (as a resource) for the scoring experiment, you can find this name in the 'update resource' help page (linked from azure portal)
30+
public const string ScoringResourceName = "Training experiment [trained model]";
31+
}
32+
}

‎Apps/ByodService/AzureMLClient/Contracts/AzureBlobDataReference.cs

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
namespace AzureMLClient.Contracts
2+
{
3+
public class AzureBlobDataReference
4+
{
5+
public string ConnectionString { get; set; }
6+
7+
public string RelativeLocation { get; set; }
8+
9+
public string BaseLocation { get; set; }
10+
11+
public string SasBlobToken { get; set; }
12+
}
13+
}

‎Apps/ByodService/AzureMLClient/Contracts/BatchRequest.cs

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
using System.Collections.Generic;
2+
3+
namespace AzureMLClient.Contracts
4+
{
5+
public class BatchRequest
6+
{
7+
public IDictionary<string, string> GlobalParameters { get; set; }
8+
9+
public AzureBlobDataReference Input { get; set; }
10+
11+
public AzureBlobDataReference Output { get; set; }
12+
}
13+
}

‎Apps/ByodService/AzureMLClient/Contracts/BatchScoreStatusCode.cs

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
namespace AzureMLClient.Contracts
2+
{
3+
public enum BatchScoreStatusCode
4+
{
5+
NotStarted,
6+
7+
Running,
8+
9+
Failed,
10+
11+
Cancelled,
12+
13+
Finished
14+
15+
}
16+
}

0 commit comments

Comments
 (0)