Skip to content

Commit 3779a71

Browse files
sayaleepotesayaleepote
committed
Working app
Added live feed detection to predict hand signs from HandSigns.mlmodel trained using Custom Vision.
1 parent 76961ad commit 3779a71

File tree

6 files changed

+110
-13
lines changed

6 files changed

+110
-13
lines changed

‎CustomVisionMicrosoftToCoreML.xcodeproj/project.pbxproj‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
4829C79620E54A1800F77F34 /* CustomVisionMicrosoftToCoreML.xcdatamodeld in Sources */ = {isa = PBXBuildFile; fileRef = 4829C79420E54A1800F77F34 /* CustomVisionMicrosoftToCoreML.xcdatamodeld */; };
1414
4829C79820E54A1900F77F34 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 4829C79720E54A1900F77F34 /* Assets.xcassets */; };
1515
4829C79B20E54A1900F77F34 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4829C79920E54A1900F77F34 /* LaunchScreen.storyboard */; };
16+
4829C7A320E54AC100F77F34 /* HandSigns.mlmodel in Sources */ = {isa = PBXBuildFile; fileRef = 4829C7A220E54AC100F77F34 /* HandSigns.mlmodel */; };
1617
/* End PBXBuildFile section */
1718

1819
/* Begin PBXFileReference section */
@@ -24,6 +25,7 @@
2425
4829C79720E54A1900F77F34 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
2526
4829C79A20E54A1900F77F34 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
2627
4829C79C20E54A1900F77F34 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
28+
4829C7A220E54AC100F77F34 /* HandSigns.mlmodel */ = {isa = PBXFileReference; lastKnownFileType = file.mlmodel; path = HandSigns.mlmodel; sourceTree = SOURCE_ROOT; };
2729
/* End PBXFileReference section */
2830

2931
/* Begin PBXFrameworksBuildPhase section */
@@ -58,6 +60,7 @@
5860
children = (
5961
4829C78D20E54A1800F77F34 /* AppDelegate.swift */,
6062
4829C78F20E54A1800F77F34 /* ViewController.swift */,
63+
4829C7A220E54AC100F77F34 /* HandSigns.mlmodel */,
6164
4829C79120E54A1800F77F34 /* Main.storyboard */,
6265
4829C79720E54A1900F77F34 /* Assets.xcassets */,
6366
4829C79920E54A1900F77F34 /* LaunchScreen.storyboard */,
@@ -138,6 +141,7 @@
138141
isa = PBXSourcesBuildPhase;
139142
buildActionMask = 2147483647;
140143
files = (
144+
4829C7A320E54AC100F77F34 /* HandSigns.mlmodel in Sources */,
141145
4829C79020E54A1800F77F34 /* ViewController.swift in Sources */,
142146
4829C78E20E54A1800F77F34 /* AppDelegate.swift in Sources */,
143147
4829C79620E54A1800F77F34 /* CustomVisionMicrosoftToCoreML.xcdatamodeld in Sources */,
@@ -284,14 +288,18 @@
284288
isa = XCBuildConfiguration;
285289
buildSettings = {
286290
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
287-
CODE_SIGN_STYLE = Automatic;
291+
CODE_SIGN_STYLE = Manual;
292+
DEVELOPMENT_TEAM = 54NTRVDHZ4;
288293
INFOPLIST_FILE = CustomVisionMicrosoftToCoreML/Info.plist;
294+
IPHONEOS_DEPLOYMENT_TARGET = 11.0;
289295
LD_RUNPATH_SEARCH_PATHS = (
290296
"$(inherited)",
291297
"@executable_path/Frameworks",
292298
);
293-
PRODUCT_BUNDLE_IDENTIFIER = com.assignment.app.CustomVisionMicrosoftToCoreML;
299+
PRODUCT_BUNDLE_IDENTIFIER = com.app.CustomVisionMicrosoftToCoreML;
294300
PRODUCT_NAME = "$(TARGET_NAME)";
301+
PROVISIONING_PROFILE = "";
302+
PROVISIONING_PROFILE_SPECIFIER = "";
295303
SWIFT_VERSION = 4.0;
296304
TARGETED_DEVICE_FAMILY = "1,2";
297305
};
@@ -301,14 +309,18 @@
301309
isa = XCBuildConfiguration;
302310
buildSettings = {
303311
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
304-
CODE_SIGN_STYLE = Automatic;
312+
CODE_SIGN_STYLE = Manual;
313+
DEVELOPMENT_TEAM = 54NTRVDHZ4;
305314
INFOPLIST_FILE = CustomVisionMicrosoftToCoreML/Info.plist;
315+
IPHONEOS_DEPLOYMENT_TARGET = 11.0;
306316
LD_RUNPATH_SEARCH_PATHS = (
307317
"$(inherited)",
308318
"@executable_path/Frameworks",
309319
);
310-
PRODUCT_BUNDLE_IDENTIFIER = com.assignment.app.CustomVisionMicrosoftToCoreML;
320+
PRODUCT_BUNDLE_IDENTIFIER = com.app.CustomVisionMicrosoftToCoreML;
311321
PRODUCT_NAME = "$(TARGET_NAME)";
322+
PROVISIONING_PROFILE = "";
323+
PROVISIONING_PROFILE_SPECIFIER = "";
312324
SWIFT_VERSION = 4.0;
313325
TARGETED_DEVICE_FAMILY = "1,2";
314326
};

‎CustomVisionMicrosoftToCoreML.xcodeproj/xcuserdata/Xoriant.xcuserdatad/xcdebugger/Breakpoints_v2.xcbkptlist‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<Bucket
3+
type = "1"
4+
version = "2.0">
5+
</Bucket>

‎CustomVisionMicrosoftToCoreML/Base.lproj/Main.storyboard‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,46 @@
11
<?xml version="1.0" encoding="UTF-8"?>
2-
<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="13122.16" systemVersion="17A277" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
2+
<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14113" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
3+
<device id="retina4_7" orientation="portrait">
4+
<adaptation id="fullscreen"/>
5+
</device>
36
<dependencies>
4-
<plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="13104.12"/>
7+
<deployment identifier="iOS"/>
8+
<plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14088"/>
59
<capability name="Safe area layout guides" minToolsVersion="9.0"/>
610
<capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
711
</dependencies>
812
<scenes>
913
<!--View Controller-->
1014
<scene sceneID="tne-QT-ifu">
1115
<objects>
12-
<viewController id="BYZ-38-t0r" customClass="ViewController" customModuleProvider="target" sceneMemberID="viewController">
16+
<viewController id="BYZ-38-t0r" customClass="ViewController" customModule="CustomVisionMicrosoftToCoreML" customModuleProvider="target" sceneMemberID="viewController">
1317
<view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
1418
<rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
1519
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
20+
<subviews>
21+
<label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" text="" textAlignment="center" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="93L-3H-2pl">
22+
<rect key="frame" x="16" y="623" width="343" height="0.0"/>
23+
<color key="backgroundColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
24+
<fontDescription key="fontDescription" type="system" pointSize="18"/>
25+
<nil key="textColor"/>
26+
<nil key="highlightedColor"/>
27+
</label>
28+
</subviews>
1629
<color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
30+
<constraints>
31+
<constraint firstItem="6Tk-OE-BBY" firstAttribute="bottom" secondItem="93L-3H-2pl" secondAttribute="bottom" constant="44" id="Rmk-mA-8Gj"/>
32+
<constraint firstItem="6Tk-OE-BBY" firstAttribute="trailing" secondItem="93L-3H-2pl" secondAttribute="trailing" constant="16" id="UeU-3s-Zmw"/>
33+
<constraint firstItem="93L-3H-2pl" firstAttribute="leading" secondItem="6Tk-OE-BBY" secondAttribute="leading" constant="16" id="VFZ-pU-K2F"/>
34+
</constraints>
1735
<viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
1836
</view>
37+
<connections>
38+
<outlet property="predictionLabel" destination="93L-3H-2pl" id="fpU-u6-xVw"/>
39+
</connections>
1940
</viewController>
2041
<placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
2142
</objects>
43+
<point key="canvasLocation" x="117.59999999999999" y="118.29085457271366"/>
2244
</scene>
2345
</scenes>
2446
</document>

‎CustomVisionMicrosoftToCoreML/Info.plist‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
<string>1</string>
2121
<key>LSRequiresIPhoneOS</key>
2222
<true/>
23+
<key>NSCameraUsageDescription</key>
24+
<string>App needs camera for detection</string>
2325
<key>UILaunchStoryboardName</key>
2426
<string>LaunchScreen</string>
2527
<key>UIMainStoryboardFile</key>
@@ -31,8 +33,6 @@
3133
<key>UISupportedInterfaceOrientations</key>
3234
<array>
3335
<string>UIInterfaceOrientationPortrait</string>
34-
<string>UIInterfaceOrientationLandscapeLeft</string>
35-
<string>UIInterfaceOrientationLandscapeRight</string>
3636
</array>
3737
<key>UISupportedInterfaceOrientations~ipad</key>
3838
<array>

‎CustomVisionMicrosoftToCoreML/ViewController.swift‎

Lines changed: 62 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,19 +7,77 @@
77
//
88

99
import UIKit
10+
import AVKit
11+
import Vision
1012

11-
class ViewController: UIViewController {
13+
enum HandSign: String {
14+
case fiveHand = "FiveHand"
15+
case fistHand = "FistHand"
16+
case victoryHand = "VictoryHand"
17+
case noHand = "NoHand"
18+
}
19+
20+
class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate {
1221

22+
@IBOutlet weak var predictionLabel: UILabel!
23+
1324
override func viewDidLoad() {
1425
super.viewDidLoad()
15-
// Do any additional setup after loading the view, typically from a nib.
26+
configureCamera()
1627
}
1728

1829
override func didReceiveMemoryWarning() {
1930
super.didReceiveMemoryWarning()
20-
// Dispose of any resources that can be recreated.
2131
}
22-
32+
33+
func configureCamera() {
34+
let captureSession = AVCaptureSession()
35+
captureSession.sessionPreset = .photo
36+
captureSession.startRunning()
37+
guard let captureDevice = AVCaptureDevice.default(for: .video) else { return }
38+
guard let captureInput = try? AVCaptureDeviceInput(device: captureDevice) else { return }
39+
captureSession.addInput(captureInput)
40+
41+
let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
42+
view.layer.addSublayer(previewLayer)
43+
previewLayer.frame = view.frame
44+
45+
let dataOutput = AVCaptureVideoDataOutput()
46+
dataOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "videoQueue"))
47+
captureSession.addOutput(dataOutput)
48+
}
49+
50+
// MARK: - AVCaptureVideoDataOutputSampleBufferDelegate
51+
52+
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
53+
guard let pixelBuffer: CVPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
54+
55+
guard let handSignsModel = try? VNCoreMLModel(for: HandSigns().model) else { return }
56+
let request = VNCoreMLRequest(model: handSignsModel) { (finishedRequest, err) in
57+
58+
guard let results = finishedRequest.results as? [VNClassificationObservation] else { return }
59+
60+
guard let firstResult = results.first else { return }
61+
var predictionString = ""
62+
DispatchQueue.main.async {
63+
switch firstResult.identifier {
64+
case HandSign.fistHand.rawValue:
65+
predictionString = "Fist👊🏽"
66+
case HandSign.victoryHand.rawValue:
67+
predictionString = "Victory✌🏽"
68+
case HandSign.fiveHand.rawValue:
69+
predictionString = "High Five🖐🏽"
70+
case HandSign.noHand.rawValue:
71+
predictionString = "No Hand ❎"
72+
default:
73+
break
74+
}
75+
self.predictionLabel.text = predictionString + "(\(firstResult.confidence))"
76+
}
77+
}
78+
79+
try? VNImageRequestHandler(cvPixelBuffer: pixelBuffer, options: [:]).perform([request])
80+
}
2381

2482
}
2583

‎HandSigns.mlmodel‎

2.77 MB
Binary file not shown.

0 commit comments

Comments
 (0)