Repository: hanleyweng/Gesture-Recognition-101-CoreML-ARKit
Branch: master
Commit: 4c73ece28e4f
Files: 13
Total size: 35.7 KB
Directory structure:
gitextract_1wv_1daf/
├── Gesture-Recognition-101-CoreML-ARKit/
│ ├── AppDelegate.swift
│ ├── Assets.xcassets/
│ │ └── AppIcon.appiconset/
│ │ └── Contents.json
│ ├── Base.lproj/
│ │ ├── LaunchScreen.storyboard
│ │ └── Main.storyboard
│ ├── Info.plist
│ ├── ViewController.swift
│ ├── art.scnassets/
│ │ └── ship.scn
│ └── example_5s0_hand_model.mlmodel
├── Gesture-Recognition-101-CoreML-ARKit.xcodeproj/
│ ├── project.pbxproj
│ ├── project.xcworkspace/
│ │ └── contents.xcworkspacedata
│ └── xcuserdata/
│ └── hanleyweng.xcuserdatad/
│ └── xcschemes/
│ └── xcschememanagement.plist
├── LICENSE
└── README.md
================================================
FILE CONTENTS
================================================
================================================
FILE: Gesture-Recognition-101-CoreML-ARKit/AppDelegate.swift
================================================
//
// AppDelegate.swift
// Gesture-Recognition-101-CoreML-ARKit
//
// Created by Hanley Weng on 10/22/17.
// Copyright © 2017 Emerging Interactions. All rights reserved.
//
import UIKit
@UIApplicationMain
class AppDelegate: UIResponder, UIApplicationDelegate {
var window: UIWindow?
func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplicationLaunchOptionsKey: Any]?) -> Bool {
// Override point for customization after application launch.
return true
}
func applicationWillResignActive(_ application: UIApplication) {
// Sent when the application is about to move from active to inactive state. This can occur for certain types of temporary interruptions (such as an incoming phone call or SMS message) or when the user quits the application and it begins the transition to the background state.
// Use this method to pause ongoing tasks, disable timers, and invalidate graphics rendering callbacks. Games should use this method to pause the game.
}
func applicationDidEnterBackground(_ application: UIApplication) {
// Use this method to release shared resources, save user data, invalidate timers, and store enough application state information to restore your application to its current state in case it is terminated later.
// If your application supports background execution, this method is called instead of applicationWillTerminate: when the user quits.
}
func applicationWillEnterForeground(_ application: UIApplication) {
// Called as part of the transition from the background to the active state; here you can undo many of the changes made on entering the background.
}
func applicationDidBecomeActive(_ application: UIApplication) {
// Restart any tasks that were paused (or not yet started) while the application was inactive. If the application was previously in the background, optionally refresh the user interface.
}
func applicationWillTerminate(_ application: UIApplication) {
// Called when the application is about to terminate. Save data if appropriate. See also applicationDidEnterBackground:.
}
}
================================================
FILE: Gesture-Recognition-101-CoreML-ARKit/Assets.xcassets/AppIcon.appiconset/Contents.json
================================================
{
"images" : [
{
"size" : "20x20",
"idiom" : "iphone",
"filename" : "Icon-40.png",
"scale" : "2x"
},
{
"size" : "20x20",
"idiom" : "iphone",
"filename" : "Icon-60.png",
"scale" : "3x"
},
{
"size" : "29x29",
"idiom" : "iphone",
"filename" : "Icon-58.png",
"scale" : "2x"
},
{
"size" : "29x29",
"idiom" : "iphone",
"filename" : "Icon-87.png",
"scale" : "3x"
},
{
"size" : "40x40",
"idiom" : "iphone",
"filename" : "Icon-80.png",
"scale" : "2x"
},
{
"size" : "40x40",
"idiom" : "iphone",
"filename" : "Icon-121.png",
"scale" : "3x"
},
{
"size" : "60x60",
"idiom" : "iphone",
"filename" : "Icon-120.png",
"scale" : "2x"
},
{
"size" : "60x60",
"idiom" : "iphone",
"filename" : "Icon-180.png",
"scale" : "3x"
},
{
"size" : "20x20",
"idiom" : "ipad",
"filename" : "Icon-20.png",
"scale" : "1x"
},
{
"size" : "20x20",
"idiom" : "ipad",
"filename" : "Icon-41.png",
"scale" : "2x"
},
{
"size" : "29x29",
"idiom" : "ipad",
"filename" : "Icon-29.png",
"scale" : "1x"
},
{
"size" : "29x29",
"idiom" : "ipad",
"filename" : "Icon-59.png",
"scale" : "2x"
},
{
"size" : "40x40",
"idiom" : "ipad",
"filename" : "Icon-42.png",
"scale" : "1x"
},
{
"size" : "40x40",
"idiom" : "ipad",
"filename" : "Icon-81.png",
"scale" : "2x"
},
{
"size" : "76x76",
"idiom" : "ipad",
"filename" : "Icon-76.png",
"scale" : "1x"
},
{
"size" : "76x76",
"idiom" : "ipad",
"filename" : "Icon-152.png",
"scale" : "2x"
},
{
"size" : "83.5x83.5",
"idiom" : "ipad",
"filename" : "Icon-167.png",
"scale" : "2x"
},
{
"size" : "1024x1024",
"idiom" : "ios-marketing",
"filename" : "Icon-1024.png",
"scale" : "1x"
}
],
"info" : {
"version" : 1,
"author" : "xcode"
}
}
================================================
FILE: Gesture-Recognition-101-CoreML-ARKit/Base.lproj/LaunchScreen.storyboard
================================================
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="13122.16" systemVersion="17A277" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
<dependencies>
<plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="13104.12"/>
<capability name="Safe area layout guides" minToolsVersion="9.0"/>
<capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
</dependencies>
<scenes>
<!--View Controller-->
<scene sceneID="EHf-IW-A2E">
<objects>
<viewController id="01J-lp-oVM" sceneMemberID="viewController">
<view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
<rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
<color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
<viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
</view>
</viewController>
<placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
</objects>
<point key="canvasLocation" x="53" y="375"/>
</scene>
</scenes>
</document>
================================================
FILE: Gesture-Recognition-101-CoreML-ARKit/Base.lproj/Main.storyboard
================================================
<?xml version="1.0" encoding="UTF-8"?>
<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="13526" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BV1-FR-VrT">
<device id="retina4_7" orientation="portrait">
<adaptation id="fullscreen"/>
</device>
<dependencies>
<deployment identifier="iOS"/>
<plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="13524"/>
<capability name="Safe area layout guides" minToolsVersion="9.0"/>
<capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
</dependencies>
<customFonts key="customFonts">
<array key="Menlo.ttc">
<string>Menlo-Regular</string>
</array>
</customFonts>
<scenes>
<!--View Controller-->
<scene sceneID="tXr-a1-R10">
<objects>
<viewController id="BV1-FR-VrT" customClass="ViewController" customModule="Gesture_Recognition_101_CoreML_ARKit" customModuleProvider="target" sceneMemberID="viewController">
<view key="view" contentMode="scaleToFill" id="Gqf-L4-38n">
<rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
<autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
<subviews>
<arscnView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="nrn-jp-KnY">
<rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
</arscnView>
<textView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" editable="NO" textAlignment="right" selectable="NO" translatesAutoresizingMaskIntoConstraints="NO" id="NTo-29-GRM">
<rect key="frame" x="0.0" y="0.0" width="375" height="100"/>
<color key="backgroundColor" white="0.0" alpha="0.51875535100000003" colorSpace="calibratedWhite"/>
<constraints>
<constraint firstAttribute="height" constant="100" id="5l8-sc-5i2"/>
</constraints>
<string key="text">Basic Gesture Recognition 101
1
2
3</string>
<color key="textColor" white="1" alpha="1" colorSpace="calibratedWhite"/>
<fontDescription key="fontDescription" name="Menlo-Regular" family="Menlo" pointSize="17"/>
<textInputTraits key="textInputTraits" autocapitalizationType="sentences"/>
</textView>
<textField opaque="NO" userInteractionEnabled="NO" contentMode="scaleToFill" contentHorizontalAlignment="center" contentVerticalAlignment="center" text="⏳" textAlignment="center" minimumFontSize="100" translatesAutoresizingMaskIntoConstraints="NO" id="pvy-DN-z9f">
<rect key="frame" x="131.5" y="423.5" width="112" height="120"/>
<nil key="textColor"/>
<fontDescription key="fontDescription" type="system" pointSize="100"/>
<textInputTraits key="textInputTraits"/>
</textField>
</subviews>
<color key="backgroundColor" white="1" alpha="1" colorSpace="calibratedWhite"/>
<constraints>
<constraint firstItem="pvy-DN-z9f" firstAttribute="centerX" secondItem="Gqf-L4-38n" secondAttribute="centerX" id="5Od-h9-GuU"/>
<constraint firstItem="NTo-29-GRM" firstAttribute="trailing" secondItem="8S3-IH-5PA" secondAttribute="trailing" id="7wW-36-aun"/>
<constraint firstItem="NTo-29-GRM" firstAttribute="leading" secondItem="8S3-IH-5PA" secondAttribute="leading" id="89i-TV-kQ0"/>
<constraint firstItem="pvy-DN-z9f" firstAttribute="centerY" secondItem="Gqf-L4-38n" secondAttribute="centerY" constant="150" id="EsY-D0-3Hg"/>
<constraint firstItem="nrn-jp-KnY" firstAttribute="top" secondItem="Gqf-L4-38n" secondAttribute="top" id="Mhf-lH-Veg"/>
<constraint firstItem="NTo-29-GRM" firstAttribute="top" secondItem="Gqf-L4-38n" secondAttribute="top" id="Vl0-jN-TkZ"/>
<constraint firstItem="nrn-jp-KnY" firstAttribute="trailing" secondItem="8S3-IH-5PA" secondAttribute="trailing" id="cgr-No-hs8"/>
<constraint firstItem="nrn-jp-KnY" firstAttribute="bottom" secondItem="8S3-IH-5PA" secondAttribute="bottom" id="ctK-pi-aZH"/>
<constraint firstItem="nrn-jp-KnY" firstAttribute="leading" secondItem="8S3-IH-5PA" secondAttribute="leading" id="plw-2B-lBO"/>
</constraints>
<viewLayoutGuide key="safeArea" id="8S3-IH-5PA"/>
</view>
<connections>
<outlet property="debugTextView" destination="NTo-29-GRM" id="hte-b1-Lo6"/>
<outlet property="sceneView" destination="nrn-jp-KnY" id="dAp-ft-qBx"/>
<outlet property="textOverlay" destination="pvy-DN-z9f" id="ffU-Xo-AHl"/>
</connections>
</viewController>
<placeholder placeholderIdentifier="IBFirstResponder" id="SZV-WD-TEh" sceneMemberID="firstResponder"/>
</objects>
<point key="canvasLocation" x="136.80000000000001" y="132.68365817091455"/>
</scene>
</scenes>
</document>
================================================
FILE: Gesture-Recognition-101-CoreML-ARKit/Info.plist
================================================
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>$(DEVELOPMENT_LANGUAGE)</string>
<key>CFBundleDisplayName</key>
<string>Gesture 🤖 101</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleShortVersionString</key>
<string>1.0</string>
<key>CFBundleVersion</key>
<string>1</string>
<key>LSRequiresIPhoneOS</key>
<true/>
<key>NSCameraUsageDescription</key>
<string>This application will use the camera for Augmented Reality.</string>
<key>UILaunchStoryboardName</key>
<string>LaunchScreen</string>
<key>UIMainStoryboardFile</key>
<string>Main</string>
<key>UIRequiredDeviceCapabilities</key>
<array>
<string>armv7</string>
<string>arkit</string>
</array>
<key>UIStatusBarHidden</key>
<true/>
<key>UISupportedInterfaceOrientations</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
<key>UISupportedInterfaceOrientations~ipad</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationPortraitUpsideDown</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
</dict>
</plist>
================================================
FILE: Gesture-Recognition-101-CoreML-ARKit/ViewController.swift
================================================
//
// ViewController.swift
// Gesture-Recognition-101-CoreML-ARKit
//
// Created by Hanley Weng on 10/22/17.
// Copyright © 2017 Emerging Interactions. All rights reserved.
//
import UIKit
import SceneKit
import ARKit
import Vision
class ViewController: UIViewController, ARSCNViewDelegate {
@IBOutlet var sceneView: ARSCNView!
@IBOutlet weak var debugTextView: UITextView!
@IBOutlet weak var textOverlay: UITextField!
let dispatchQueueML = DispatchQueue(label: "com.hw.dispatchqueueml") // A Serial Queue
var visionRequests = [VNRequest]()
override func viewDidLoad() {
super.viewDidLoad()
// --- ARKIT ---
// Set the view's delegate
sceneView.delegate = self
// Show statistics such as fps and timing information
sceneView.showsStatistics = true
// Create a new scene
let scene = SCNScene() // SCNScene(named: "art.scnassets/ship.scn")!
// Set the scene to the view
sceneView.scene = scene
// --- ML & VISION ---
// Setup Vision Model
guard let selectedModel = try? VNCoreMLModel(for: example_5s0_hand_model().model) else {
fatalError("Could not load model. Ensure model has been drag and dropped (copied) to XCode Project. Also ensure the model is part of a target (see: https://stackoverflow.com/questions/45884085/model-is-not-part-of-any-target-add-the-model-to-a-target-to-enable-generation ")
}
// Set up Vision-CoreML Request
let classificationRequest = VNCoreMLRequest(model: selectedModel, completionHandler: classificationCompleteHandler)
classificationRequest.imageCropAndScaleOption = VNImageCropAndScaleOption.centerCrop // Crop from centre of images and scale to appropriate size.
visionRequests = [classificationRequest]
// Begin Loop to Update CoreML
loopCoreMLUpdate()
}
override func viewWillAppear(_ animated: Bool) {
super.viewWillAppear(animated)
// Create a session configuration
let configuration = ARWorldTrackingConfiguration()
// Run the view's session
sceneView.session.run(configuration)
}
override func viewWillDisappear(_ animated: Bool) {
super.viewWillDisappear(animated)
// Pause the view's session
sceneView.session.pause()
}
override func didReceiveMemoryWarning() {
super.didReceiveMemoryWarning()
// Release any cached data, images, etc that aren't in use.
}
// MARK: - ARSCNViewDelegate
func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) {
DispatchQueue.main.async {
// Do any desired updates to SceneKit here.
}
}
// MARK: - MACHINE LEARNING
func loopCoreMLUpdate() {
// Continuously run CoreML whenever it's ready. (Preventing 'hiccups' in Frame Rate)
dispatchQueueML.async {
// 1. Run Update.
self.updateCoreML()
// 2. Loop this function.
self.loopCoreMLUpdate()
}
}
func updateCoreML() {
// Get Camera Image as RGB
let pixbuff : CVPixelBuffer? = (sceneView.session.currentFrame?.capturedImage)
if pixbuff == nil { return }
let ciImage = CIImage(cvPixelBuffer: pixbuff!)
// Prepare CoreML/Vision Request
let imageRequestHandler = VNImageRequestHandler(ciImage: ciImage, options: [:])
// Run Vision Image Request
do {
try imageRequestHandler.perform(self.visionRequests)
} catch {
print(error)
}
}
func classificationCompleteHandler(request: VNRequest, error: Error?) {
// Catch Errors
if error != nil {
print("Error: " + (error?.localizedDescription)!)
return
}
guard let observations = request.results else {
print("No results")
return
}
// Get Classifications
let classifications = observations[0...2] // top 3 results
.flatMap({ $0 as? VNClassificationObservation })
.map({ "\($0.identifier) \(String(format:" : %.2f", $0.confidence))" })
.joined(separator: "\n")
// Render Classifications
DispatchQueue.main.async {
// Print Classifications
// print(classifications)
// print("-------------")
// Display Debug Text on screen
self.debugTextView.text = "TOP 3 PROBABILITIES: \n" + classifications
// Display Top Symbol
var symbol = "❎"
let topPrediction = classifications.components(separatedBy: "\n")[0]
let topPredictionName = topPrediction.components(separatedBy: ":")[0].trimmingCharacters(in: .whitespaces)
// Only display a prediction if confidence is above 1%
let topPredictionScore:Float? = Float(topPrediction.components(separatedBy: ":")[1].trimmingCharacters(in: .whitespaces))
if (topPredictionScore != nil && topPredictionScore! > 0.01) {
if (topPredictionName == "fist-UB-RHand") { symbol = "👊" }
if (topPredictionName == "FIVE-UB-RHand") { symbol = "🖐" }
}
self.textOverlay.text = symbol
}
}
// MARK: - HIDE STATUS BAR
override var prefersStatusBarHidden : Bool { return true }
}
================================================
FILE: Gesture-Recognition-101-CoreML-ARKit.xcodeproj/project.pbxproj
================================================
// !$*UTF8*$!
{
archiveVersion = 1;
classes = {
};
objectVersion = 48;
objects = {
/* Begin PBXBuildFile section */
AE85734E1F9D179100D73806 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = AE85734D1F9D179100D73806 /* AppDelegate.swift */; };
AE8573501F9D179100D73806 /* art.scnassets in Resources */ = {isa = PBXBuildFile; fileRef = AE85734F1F9D179100D73806 /* art.scnassets */; };
AE8573521F9D179100D73806 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = AE8573511F9D179100D73806 /* ViewController.swift */; };
AE8573551F9D179100D73806 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = AE8573531F9D179100D73806 /* Main.storyboard */; };
AE8573571F9D179100D73806 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = AE8573561F9D179100D73806 /* Assets.xcassets */; };
AE85735A1F9D179100D73806 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = AE8573581F9D179100D73806 /* LaunchScreen.storyboard */; };
AE8573651F9D206800D73806 /* example_5s0_hand_model.mlmodel in Sources */ = {isa = PBXBuildFile; fileRef = AE8573641F9D206800D73806 /* example_5s0_hand_model.mlmodel */; };
/* End PBXBuildFile section */
/* Begin PBXFileReference section */
AE85734A1F9D179100D73806 /* Gesture-Recognition-101-CoreML-ARKit.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "Gesture-Recognition-101-CoreML-ARKit.app"; sourceTree = BUILT_PRODUCTS_DIR; };
AE85734D1F9D179100D73806 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
AE85734F1F9D179100D73806 /* art.scnassets */ = {isa = PBXFileReference; lastKnownFileType = wrapper.scnassets; path = art.scnassets; sourceTree = "<group>"; };
AE8573511F9D179100D73806 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = "<group>"; };
AE8573541F9D179100D73806 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = "<group>"; };
AE8573561F9D179100D73806 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
AE8573591F9D179100D73806 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
AE85735B1F9D179100D73806 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
AE8573641F9D206800D73806 /* example_5s0_hand_model.mlmodel */ = {isa = PBXFileReference; lastKnownFileType = file.mlmodel; path = example_5s0_hand_model.mlmodel; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
AE8573471F9D179100D73806 /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXFrameworksBuildPhase section */
/* Begin PBXGroup section */
AE8573411F9D179100D73806 = {
isa = PBXGroup;
children = (
AE85734C1F9D179100D73806 /* Gesture-Recognition-101-CoreML-ARKit */,
AE85734B1F9D179100D73806 /* Products */,
);
sourceTree = "<group>";
};
AE85734B1F9D179100D73806 /* Products */ = {
isa = PBXGroup;
children = (
AE85734A1F9D179100D73806 /* Gesture-Recognition-101-CoreML-ARKit.app */,
);
name = Products;
sourceTree = "<group>";
};
AE85734C1F9D179100D73806 /* Gesture-Recognition-101-CoreML-ARKit */ = {
isa = PBXGroup;
children = (
AE85734D1F9D179100D73806 /* AppDelegate.swift */,
AE85734F1F9D179100D73806 /* art.scnassets */,
AE8573511F9D179100D73806 /* ViewController.swift */,
AE8573531F9D179100D73806 /* Main.storyboard */,
AE8573561F9D179100D73806 /* Assets.xcassets */,
AE8573581F9D179100D73806 /* LaunchScreen.storyboard */,
AE85735B1F9D179100D73806 /* Info.plist */,
AE8573641F9D206800D73806 /* example_5s0_hand_model.mlmodel */,
);
path = "Gesture-Recognition-101-CoreML-ARKit";
sourceTree = "<group>";
};
/* End PBXGroup section */
/* Begin PBXNativeTarget section */
AE8573491F9D179100D73806 /* Gesture-Recognition-101-CoreML-ARKit */ = {
isa = PBXNativeTarget;
buildConfigurationList = AE85735E1F9D179100D73806 /* Build configuration list for PBXNativeTarget "Gesture-Recognition-101-CoreML-ARKit" */;
buildPhases = (
AE8573461F9D179100D73806 /* Sources */,
AE8573471F9D179100D73806 /* Frameworks */,
AE8573481F9D179100D73806 /* Resources */,
);
buildRules = (
);
dependencies = (
);
name = "Gesture-Recognition-101-CoreML-ARKit";
productName = "Gesture-Recognition-101-CoreML-ARKit";
productReference = AE85734A1F9D179100D73806 /* Gesture-Recognition-101-CoreML-ARKit.app */;
productType = "com.apple.product-type.application";
};
/* End PBXNativeTarget section */
/* Begin PBXProject section */
AE8573421F9D179100D73806 /* Project object */ = {
isa = PBXProject;
attributes = {
LastSwiftUpdateCheck = 0910;
LastUpgradeCheck = 0910;
ORGANIZATIONNAME = "Emerging Interactions";
TargetAttributes = {
AE8573491F9D179100D73806 = {
CreatedOnToolsVersion = 9.1;
ProvisioningStyle = Automatic;
};
};
};
buildConfigurationList = AE8573451F9D179100D73806 /* Build configuration list for PBXProject "Gesture-Recognition-101-CoreML-ARKit" */;
compatibilityVersion = "Xcode 8.0";
developmentRegion = en;
hasScannedForEncodings = 0;
knownRegions = (
en,
Base,
);
mainGroup = AE8573411F9D179100D73806;
productRefGroup = AE85734B1F9D179100D73806 /* Products */;
projectDirPath = "";
projectRoot = "";
targets = (
AE8573491F9D179100D73806 /* Gesture-Recognition-101-CoreML-ARKit */,
);
};
/* End PBXProject section */
/* Begin PBXResourcesBuildPhase section */
AE8573481F9D179100D73806 /* Resources */ = {
isa = PBXResourcesBuildPhase;
buildActionMask = 2147483647;
files = (
AE8573501F9D179100D73806 /* art.scnassets in Resources */,
AE85735A1F9D179100D73806 /* LaunchScreen.storyboard in Resources */,
AE8573571F9D179100D73806 /* Assets.xcassets in Resources */,
AE8573551F9D179100D73806 /* Main.storyboard in Resources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXResourcesBuildPhase section */
/* Begin PBXSourcesBuildPhase section */
AE8573461F9D179100D73806 /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
AE8573521F9D179100D73806 /* ViewController.swift in Sources */,
AE85734E1F9D179100D73806 /* AppDelegate.swift in Sources */,
AE8573651F9D206800D73806 /* example_5s0_hand_model.mlmodel in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXSourcesBuildPhase section */
/* Begin PBXVariantGroup section */
AE8573531F9D179100D73806 /* Main.storyboard */ = {
isa = PBXVariantGroup;
children = (
AE8573541F9D179100D73806 /* Base */,
);
name = Main.storyboard;
sourceTree = "<group>";
};
AE8573581F9D179100D73806 /* LaunchScreen.storyboard */ = {
isa = PBXVariantGroup;
children = (
AE8573591F9D179100D73806 /* Base */,
);
name = LaunchScreen.storyboard;
sourceTree = "<group>";
};
/* End PBXVariantGroup section */
/* Begin XCBuildConfiguration section */
AE85735C1F9D179100D73806 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_ANALYZER_NONNULL = YES;
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
CLANG_CXX_LIBRARY = "libc++";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_COMMA = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INFINITE_RECURSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
CLANG_WARN_STRICT_PROTOTYPES = YES;
CLANG_WARN_SUSPICIOUS_MOVE = YES;
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
CODE_SIGN_IDENTITY = "iPhone Developer";
COPY_PHASE_STRIP = NO;
DEBUG_INFORMATION_FORMAT = dwarf;
ENABLE_STRICT_OBJC_MSGSEND = YES;
ENABLE_TESTABILITY = YES;
GCC_C_LANGUAGE_STANDARD = gnu11;
GCC_DYNAMIC_NO_PIC = NO;
GCC_NO_COMMON_BLOCKS = YES;
GCC_OPTIMIZATION_LEVEL = 0;
GCC_PREPROCESSOR_DEFINITIONS = (
"DEBUG=1",
"$(inherited)",
);
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 11.1;
MTL_ENABLE_DEBUG_INFO = YES;
ONLY_ACTIVE_ARCH = YES;
SDKROOT = iphoneos;
SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
};
name = Debug;
};
AE85735D1F9D179100D73806 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_ANALYZER_NONNULL = YES;
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
CLANG_CXX_LIBRARY = "libc++";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_COMMA = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INFINITE_RECURSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
CLANG_WARN_STRICT_PROTOTYPES = YES;
CLANG_WARN_SUSPICIOUS_MOVE = YES;
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
CODE_SIGN_IDENTITY = "iPhone Developer";
COPY_PHASE_STRIP = NO;
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
ENABLE_NS_ASSERTIONS = NO;
ENABLE_STRICT_OBJC_MSGSEND = YES;
GCC_C_LANGUAGE_STANDARD = gnu11;
GCC_NO_COMMON_BLOCKS = YES;
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 11.1;
MTL_ENABLE_DEBUG_INFO = NO;
SDKROOT = iphoneos;
SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule";
VALIDATE_PRODUCT = YES;
};
name = Release;
};
AE85735F1F9D179100D73806 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CODE_SIGN_STYLE = Automatic;
DEVELOPMENT_TEAM = 8KS4DU4Y2K;
INFOPLIST_FILE = "Gesture-Recognition-101-CoreML-ARKit/Info.plist";
IPHONEOS_DEPLOYMENT_TARGET = 11.0;
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
PRODUCT_BUNDLE_IDENTIFIER = "com.emerging-interactions.Gesture-Recognition-101-CoreML-ARKit";
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_VERSION = 4.0;
TARGETED_DEVICE_FAMILY = "1,2";
};
name = Debug;
};
AE8573601F9D179100D73806 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CODE_SIGN_STYLE = Automatic;
DEVELOPMENT_TEAM = 8KS4DU4Y2K;
INFOPLIST_FILE = "Gesture-Recognition-101-CoreML-ARKit/Info.plist";
IPHONEOS_DEPLOYMENT_TARGET = 11.0;
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
PRODUCT_BUNDLE_IDENTIFIER = "com.emerging-interactions.Gesture-Recognition-101-CoreML-ARKit";
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_VERSION = 4.0;
TARGETED_DEVICE_FAMILY = "1,2";
};
name = Release;
};
/* End XCBuildConfiguration section */
/* Begin XCConfigurationList section */
AE8573451F9D179100D73806 /* Build configuration list for PBXProject "Gesture-Recognition-101-CoreML-ARKit" */ = {
isa = XCConfigurationList;
buildConfigurations = (
AE85735C1F9D179100D73806 /* Debug */,
AE85735D1F9D179100D73806 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
AE85735E1F9D179100D73806 /* Build configuration list for PBXNativeTarget "Gesture-Recognition-101-CoreML-ARKit" */ = {
isa = XCConfigurationList;
buildConfigurations = (
AE85735F1F9D179100D73806 /* Debug */,
AE8573601F9D179100D73806 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
/* End XCConfigurationList section */
};
rootObject = AE8573421F9D179100D73806 /* Project object */;
}
================================================
FILE: Gesture-Recognition-101-CoreML-ARKit.xcodeproj/project.xcworkspace/contents.xcworkspacedata
================================================
<?xml version="1.0" encoding="UTF-8"?>
<Workspace
version = "1.0">
<FileRef
location = "self:Gesture-Recognition-101-CoreML-ARKit.xcodeproj">
</FileRef>
</Workspace>
================================================
FILE: Gesture-Recognition-101-CoreML-ARKit.xcodeproj/xcuserdata/hanleyweng.xcuserdatad/xcschemes/xcschememanagement.plist
================================================
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>SchemeUserState</key>
<dict>
<key>Gesture-Recognition-101-CoreML-ARKit.xcscheme</key>
<dict>
<key>orderHint</key>
<integer>0</integer>
</dict>
</dict>
</dict>
</plist>
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2017 Hanley
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
# Hand Gesture Recognition
This simple sample project recognizes hands in realtime. 👋 It serves as a basic example for recognizing your own objects. Suitable for AR 🤓. Written for the tutorial [“Create your own Object Recognizer”](https://medium.com/p/7f8c09b461a1).
[Demo Video - on Youtube](https://youtu.be/P3Q8awgT9Lk)
Tech: iOS 11, ARKit, CoreML, iPhone 7 plus, Xcode 9.1, Swift 4.0
## Notes:
This demonstrates basic Object Recognition (for spread hand 🖐, fist 👊, and no hands ❎). It serves as a building block for object detection, localization, gesture-recognition, and hand tracking.
Disclaimer:
The sample model provided here was captured in 1 hour and is biased to one human hand 👋🏼. It’s intended as a placeholder for your own models. (See [Tutorial](https://medium.com/p/7f8c09b461a1))
---
## Steps Taken (Overview)
_Here’s an overview of the steps taken. (You can also view my commit history to see steps involved.)_
1. Build an Intuition by playing with Google CL's [Teachable Machine](https://teachablemachine.withgoogle.com/).
2. Build dataset.
3. Create a Core ML Model using Microsoft's [CustomVision.ai](https://www.customvision.ai/).
4. Run the model in realtime with ARKit.
[Full Tutorial here](https://medium.com/p/7f8c09b461a1)
_P.S. A few well selected images are sufficient for CustomVision.ai . For the sample model here, I did 3 rounds of data collection (adding 63, 38, 21 images per round). Alternating classes during data collection also appeared to work better than gathering all the class images at once._
## License
MIT Open Source License. 🧞 Use as you wish. Have fun! 😁
gitextract_1wv_1daf/ ├── Gesture-Recognition-101-CoreML-ARKit/ │ ├── AppDelegate.swift │ ├── Assets.xcassets/ │ │ └── AppIcon.appiconset/ │ │ └── Contents.json │ ├── Base.lproj/ │ │ ├── LaunchScreen.storyboard │ │ └── Main.storyboard │ ├── Info.plist │ ├── ViewController.swift │ ├── art.scnassets/ │ │ └── ship.scn │ └── example_5s0_hand_model.mlmodel ├── Gesture-Recognition-101-CoreML-ARKit.xcodeproj/ │ ├── project.pbxproj │ ├── project.xcworkspace/ │ │ └── contents.xcworkspacedata │ └── xcuserdata/ │ └── hanleyweng.xcuserdatad/ │ └── xcschemes/ │ └── xcschememanagement.plist ├── LICENSE └── README.md
Condensed preview — 13 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (40K chars).
[
{
"path": "Gesture-Recognition-101-CoreML-ARKit/AppDelegate.swift",
"chars": 2210,
"preview": "//\n// AppDelegate.swift\n// Gesture-Recognition-101-CoreML-ARKit\n//\n// Created by Hanley Weng on 10/22/17.\n// Copyrig"
},
{
"path": "Gesture-Recognition-101-CoreML-ARKit/Assets.xcassets/AppIcon.appiconset/Contents.json",
"chars": 2209,
"preview": "{\n \"images\" : [\n {\n \"size\" : \"20x20\",\n \"idiom\" : \"iphone\",\n \"filename\" : \"Icon-40.png\",\n \"scale\""
},
{
"path": "Gesture-Recognition-101-CoreML-ARKit/Base.lproj/LaunchScreen.storyboard",
"chars": 1681,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<document type=\"com.apple.InterfaceBuilder3.CocoaTouch.Storyboard"
},
{
"path": "Gesture-Recognition-101-CoreML-ARKit/Base.lproj/Main.storyboard",
"chars": 6000,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<document type=\"com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB\" version=\"3"
},
{
"path": "Gesture-Recognition-101-CoreML-ARKit/Info.plist",
"chars": 1706,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/P"
},
{
"path": "Gesture-Recognition-101-CoreML-ARKit/ViewController.swift",
"chars": 5653,
"preview": "//\n// ViewController.swift\n// Gesture-Recognition-101-CoreML-ARKit\n//\n// Created by Hanley Weng on 10/22/17.\n// Copy"
},
{
"path": "Gesture-Recognition-101-CoreML-ARKit.xcodeproj/project.pbxproj",
"chars": 13660,
"preview": "// !$*UTF8*$!\n{\n\tarchiveVersion = 1;\n\tclasses = {\n\t};\n\tobjectVersion = 48;\n\tobjects = {\n\n/* Begin PBXBuildFile section *"
},
{
"path": "Gesture-Recognition-101-CoreML-ARKit.xcodeproj/project.xcworkspace/contents.xcworkspacedata",
"chars": 181,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Workspace\n version = \"1.0\">\n <FileRef\n location = \"self:Gesture-Recogni"
},
{
"path": "Gesture-Recognition-101-CoreML-ARKit.xcodeproj/xcuserdata/hanleyweng.xcuserdatad/xcschemes/xcschememanagement.plist",
"chars": 359,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/P"
},
{
"path": "LICENSE",
"chars": 1063,
"preview": "MIT License\n\nCopyright (c) 2017 Hanley\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof "
},
{
"path": "README.md",
"chars": 1813,
"preview": "# Hand Gesture Recognition\nThis simple sample project recognizes hands in realtime. 👋 It serves as a basic example for r"
}
]
// ... and 2 more files (download for full content)
About this extraction
This page contains the full source code of the hanleyweng/Gesture-Recognition-101-CoreML-ARKit GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 13 files (35.7 KB), approximately 10.7k tokens. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.