video matching project

.gitignore

@@ -0,0 +1,15 @@
+*.rgb
+.ipynb_checkpoints/
+.DS_Store
+.idea/
+*/__pycache__/
+*/.pytest_cache/
+database_videos/
+database_videos_jpeg/
+query/
+CS576FinalProjData/
+query_videos/
+*.png
+*.h5
+jupyter_notebooks/videoData/
+*.mov

README.md

@@ -0,0 +1,39 @@
+# Video Matching
+
+## Step 0. Watch the demo :)
+[![](http://img.youtube.com/vi/fO46zt5-JNs/0.jpg)](http://www.youtube.com/watch?v=fO46zt5-JNs "Video Matching")
+
+## Step 1. Take a look at Jupyter notebooks
+You can take a look at Jupyter notebooks in /jupyter_notebooks which contains color matching part and object matching part. They were wrote solely by me and the software is basically putting them together as a program with user interface. I also wrote comprehensive documentation/comments in those notebooks so you should have no problem understanding what I'm doing.
+
+## Step 2. Get your video database and model resource
+
+First, you should store a video database with jpg format and add that directory into `config.json`. 
+The directory within `database_videos` should be like 
+```angular2html
+.
+├── flowers
+├── interview
+├── movie
+├── musicvideo
+├── sports
+├── starcraft
+└── traffic
+```
+Each of the directory should include an audio file, and a list of images format with `FILENAME + FRAME_NUMBER.jpg`
+For convenience, query video should be organized in a similar way, but it's also compatible with rgb files.
+
+You need to add the directory of resource videos into `config.json`. 
+Links to download database/query videos:
+[Database](https://drive.google.com/file/d/1oHsvXNuoni_aVqi1qhY563X6TtmepSkE/view?usp=sharing)
+
+[Query](https://drive.google.com/file/d/1rf8JRYKSG3UnGRKkoOCX10FGI0Z_UBrX/view?usp=sharing)
+
+Another important configuration is the `resource` directory path. Since we will use yolo to detect objects, it's important to put an object detection model traning network under `resource/object_detection_model`. 
+
+[Link to download object detection model](https://drive.google.com/file/d/1_eL2UrnNOHkNcyYLOGAb9FYyKRD2gGq6/view?usp=sharing)
+(Side note: yolov3 works better)
+
+## Step 3. Run `main.py`
+
+`main.py` is the entry point for the program. It would take a while to load all database images and a tensorflow trained network. After finish loading, you can upload the query video directory to compare and search for match videos.  

config.json

@@ -0,0 +1,4 @@
+{
+  "jpgDatabaseDir" : "full_path_of_the_dir_that_contains_database_videos",
+  "resourceBaseDir": "full_path_of_the_dir_that_contains_query_videos"
+}

gui/EmbedGraph.py

@@ -0,0 +1,79 @@
+import sys
+import numpy as np
+import PyQt5
+from matplotlib.backends.qt_compat import QtCore, QtWidgets
+from matplotlib.backends.backend_qt5agg import FigureCanvas
+from numpy.linalg import norm
+from matplotlib.figure import Figure
+from preprocess.entropyChecker import EntropyChecker
+
+class EmbedGraph(QtWidgets.QWidget):
+
+    def __init__(self, parent):
+        super().__init__()
+        self.static_canvas = FigureCanvas(Figure(figsize=(5, 3)))
+        self.layout = QtWidgets.QVBoxLayout(parent)
+        self.layout.addWidget(self.static_canvas)
+
+
+    def addData(self, y, startPos, queryY):
+        for i in range(self.layout.count()):
+            self.layout.itemAt(i).widget().close()
+        self.static_canvas = FigureCanvas(Figure())
+        self.static_canvas.figure.subplots_adjust(left=0, right=1, top=1, bottom=0)
+        self.normalized_canvas = FigureCanvas(Figure())
+        self.normalized_canvas.figure.subplots_adjust(left=0, right=1, top=1, bottom=0)
+        self.layout.addWidget(self.static_canvas)
+        self.layout.addWidget(self.normalized_canvas)
+        self._static_ax = self.static_canvas.figure.subplots()
+        self._normal_ax = self.normalized_canvas.figure.subplots()
+        self._static_ax.plot(np.linspace(0, len(y), len(y)), y, "-")
+        self._static_ax.plot(np.linspace(startPos, startPos + len(queryY), len(queryY)),
+                             queryY, "-")
+        self._static_ax.spines['right'].set_color('none')
+        self._static_ax.spines['top'].set_color('none')
+        self._static_ax.axis('off')
+        self._static_ax.set_autoscale_on(True)
+
+
+        normalized_targetY = y[startPos:startPos + len(queryY)]
+        normalized_targetY = normalized_targetY / np.sqrt(np.sum(normalized_targetY ** 2))
+        self._normal_ax.plot(np.linspace(0, len(queryY), len(queryY)), normalized_targetY, "-")
+        normalized_queryY = queryY / np.sqrt(np.sum(queryY ** 2))
+        self._normal_ax.plot(np.linspace(0, len(queryY), len(queryY)),
+                             normalized_queryY, "-")
+        self._normal_ax.spines['right'].set_color('none')
+        self._normal_ax.spines['top'].set_color('none')
+        self._normal_ax.axis('off')
+        self._normal_ax.set_autoscale_on(True)
+
+if __name__ == "__main__":
+    qapp = QtWidgets.QApplication(sys.argv)
+    matchHeatGraph = PyQt5.QtWidgets.QGraphicsView()
+    matchHeatGraph.setGeometry(QtCore.QRect(0, 0, 351, 351))
+    # matchHeatGraph.setWidgetResizable(True)
+    matchHeatGraph.setObjectName("matchHeatGraph")
+    # layout = QtWidgets.QVBoxLayout(matchHeatGraph)
+    #
+    # graph = EmbedGraph(matchHeatGraph)
+    # graph.addData(np.tan(np.linspace(0, 600, 600)))
+    layout = QtWidgets.QVBoxLayout(matchHeatGraph)
+    static_canvas = FigureCanvas(Figure())
+    static_canvas.figure.subplots_adjust(left=0, right=1, top=1, bottom=0)
+    layout.addWidget(static_canvas)
+    axis = static_canvas.figure.subplots()
+    x = np.linspace(0, 600, 600)
+    y = np.tan(x)
+    z = np.sin(np.linspace(0, 150, 150))
+    axis.plot(x, y, "-")
+    axis.plot(np.linspace(80, 80 + 150, 150), z, "-")
+    # static_canvas.show()
+    axis.axis('off')
+    axis.set_autoscale_on(True)
+    axis.set_xmargin(0)
+    axis.set_ymargin(0)
+    matchHeatGraph.show()
+    # graph.addData(np.square(np.linspace(0, 600, 600)))
+    # graph.addData(np.tan(np.linspace(0, 600, 600)))
+
+    qapp.exec_()

gui/VideoLoadingManager.py

@@ -0,0 +1,47 @@
+from PyQt5 import QtCore
+from PyQt5.QtGui import QImage
+import time
+
+class VideoLoadingManager(QtCore.QObject):
+    imageChanged = QtCore.pyqtSignal((QImage, int))
+
+    def __init__(self):
+        QtCore.QObject.__init__(self)
+        self.index = 0
+        self.running = False
+        self.prevTime = time.time()
+
+    def setVideoLoader(self, imgLoader):
+        self.videoLoader = imgLoader
+
+    def loadingImgs(self):
+        while True:
+            curTime = time.time()
+            if self.running and self.index < self.videoLoader.getVideoLenth() and curTime > self.prevTime:
+                img = self.videoLoader.getImgAt(self.index)
+                self.imageChanged.emit(img, self.index)
+                self.index += 1
+                self.prevTime += (1000 / 30) / 1000
+
+    def play(self):
+        self.running = True
+        self.prevTime = time.time()
+
+    def pause(self):
+        self.running = False
+
+    def stop(self):
+        self.running = False
+        self.index = 0
+
+    def setFramePos(self, pos):
+        self.index = pos
+
+    def getVideoLength(self):
+        return self.videoLoader.getVideoLenth()
+
+    def getOneFrame(self, pos):
+        return self.videoLoader.getImgAt(pos)
+
+    def existVideo(self):
+        return hasattr(self, 'videoLoader')

gui/imageShower.py

@@ -0,0 +1,140 @@
+import sys
+from PyQt5 import QtCore, QtMultimedia
+from PyQt5.QtWidgets import QWidget, QGridLayout, QGraphicsView, QGraphicsScene, QGraphicsPixmapItem, QApplication, QScrollArea
+from PyQt5.QtMultimedia import QMediaPlayer
+from PyQt5.QtGui import QImage, QPixmap
+from videoReader.videoReader import VideoReader
+from gui.VideoLoadingManager import VideoLoadingManager
+
+class ImageShower(QWidget):
+
+    def __init__(self, parent=None):
+        QWidget.__init__(self, parent)
+        self.__videoReader = VideoLoadingManager()
+        lay = QGridLayout(self)
+        gv = QGraphicsView()
+        lay.addWidget(gv)
+        scene = QGraphicsScene(self)
+        gv.setScene(scene)
+        gv.setContentsMargins(0, 0, 0, 0)
+        self.pixmap_item = QGraphicsPixmapItem()
+        self.player = QtMultimedia.QMediaPlayer(None, QMediaPlayer.VideoSurface)
+        # self.player.setPlaybackRate(0.925)
+        scene.addItem(self.pixmap_item)
+        gv.fitInView(scene.itemsBoundingRect(), QtCore.Qt.KeepAspectRatio)
+        self.workerThread = QtCore.QThread()
+        self.__videoReader.moveToThread(self.workerThread)
+        self.workerThread.finished.connect(self.__videoReader.deleteLater)
+        self.workerThread.started.connect(self.__videoReader.loadingImgs)
+        self.__videoReader.imageChanged.connect(self.setImage)
+        self.colorPaleteCallback = None
+    def setupCallback(self, callback):
+        self.callback = callback
+
+    @QtCore.pyqtSlot(QImage, int)
+    def setImage(self, image, index):
+        pixmap = QPixmap.fromImage(image)
+        self.pixmap_item.setPixmap(pixmap)
+        # position = self.player.position()
+        # imgRate = (index / 600) * 100
+        # audioRate = (position / 20010) * 100
+        # print("{0:.2f}, {1:.2f}, position: {2:d}".format(imgRate, audioRate, position))
+
+    def play(self):
+        self.workerThread.start()
+        self.__videoReader.play()
+        self.player.play()
+
+    def setupCallback(self, callback):
+        self.colorPaleteCallback = callback
+
+    def setupVideo(self, videoPath):
+        videoReader = VideoReader(videoPath)
+        videoReader.loadImgs()
+        self.__videoReader.setVideoLoader(videoReader)
+        print("video path: " + videoReader.getVideoPath())
+        # setup audio
+        fullPath = QtCore.QDir(videoReader.getVideoPath()).absoluteFilePath(videoReader.getVideoName() + ".wav")
+        url = QtCore.QUrl.fromLocalFile(fullPath)
+        content = QtMultimedia.QMediaContent(url)
+        self.player = QtMultimedia.QMediaPlayer()
+        # self.player.setPlaybackRate(0.925)
+        self.player.setMedia(content)
+        self.stop()
+
+    def setupVideoFromImgSeq(self, videoPath, imgSeq):
+        videoReader = VideoReader(videoPath)
+        videoReader.addVideoFrames(imgSeq)
+        self.__videoReader.setVideoLoader(videoReader)
+        print("video path: " + videoReader.getVideoPath())
+        # setup audio
+        fullPath = QtCore.QDir(videoReader.getVideoPath()).absoluteFilePath(videoReader.getVideoName() + ".wav")
+        url = QtCore.QUrl.fromLocalFile(fullPath)
+        content = QtMultimedia.QMediaContent(url)
+        self.player = QtMultimedia.QMediaPlayer()
+        # self.player.setPlaybackRate(0.925)
+        self.player.setMedia(content)
+        self.stop()
+
+    def setupVideoReader(self, videoReader):
+        self.__videoReader.setVideoLoader(videoReader)
+        fullPath = QtCore.QDir(videoReader.getVideoPath()).absoluteFilePath(videoReader.getVideoName() + ".wav")
+        url = QtCore.QUrl.fromLocalFile(fullPath)
+        content = QtMultimedia.QMediaContent(url)
+        print("current video is: " + videoReader.getVideoName())
+        self.player = QtMultimedia.QMediaPlayer()
+        # self.player.setPlaybackRate(0.925)
+        self.player.setMedia(content)
+        self.stop()
+
+    def pause(self):
+        if self.player.state() == QMediaPlayer.PlayingState:
+            self.player.pause()
+            self.__videoReader.pause()
+        else:
+            self.player.play()
+            self.__videoReader.play()
+
+    def stop(self):
+        self.player.stop()
+        self.__videoReader.stop()
+        self.workerThread.exit()
+
+    def changeFrame(self, position):
+        # It will never sync when you dragging the horizonalBar
+        # print("current position is: {}".format(position))
+        self.player.pause()
+        self.__videoReader.pause()
+        videoLen = self.__videoReader.getVideoLength()
+
+        # self.player.setPosition(200 * position)
+        # position = self.player.position()
+        frameNum = int(6 * position)
+        if self.colorPaleteCallback is not None:
+            self.colorPaleteCallback(frameNum)
+        pixmap = QPixmap.fromImage(self.fetchOneFrame(frameNum))
+        self.pixmap_item.setPixmap(pixmap)
+        self.__videoReader.setFramePos(frameNum)
+        self.player.setPosition(200 * position)
+
+    # def bindControl(self, bar):
+    #     def changePositon(position):
+    #         print("current position is: " + str(position))
+    #     self.player.positionChanged.connect(changePositon)
+
+    def fetchOneFrame(self, position):
+        return self.__videoReader.getOneFrame(position)
+
+    def existVideo(self):
+        return self.__videoReader is not None and self.__videoReader.existVideo()
+if __name__ == "__main__":
+    app = QApplication(sys.argv)
+    #scroll = QScrollArea()
+    # scroll.setGeometry(QtCore.QRect(0, 0, 400, 300))
+
+    w = ImageShower()
+    w.setGeometry(QtCore.QRect(0, 0, 378, 314))
+    w.show()
+    w.setupVideo("/Users/weiwang/Desktop/CSCI576/project/CSCI576-Final-Project/database_videos/traffic")
+    w.play()
+    sys.exit(app.exec_())

gui/resource/README.md

@@ -0,0 +1,2 @@
+To create UI from Qt Design
+`exec python3 -m PyQt5.uic.pyuic fileName -o targetFileName -x`