Repository: LukasBommes/mv-extractor Branch: master Commit: 7d8860e7d276 Files: 1038 Total size: 105.8 KB Directory structure: gitextract_5c5m18h4/ ├── .github/ │ └── workflows/ │ └── ci.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── README.md ├── dockerhub.md ├── extract_mvs.py ├── pyproject.toml ├── release.md ├── run.sh ├── setup.py ├── src/ │ └── mvextractor/ │ ├── __init__.py │ ├── __main__.py │ ├── mat_to_ndarray.cpp │ ├── mat_to_ndarray.hpp │ ├── py_video_cap.cpp │ ├── pycompat.hpp │ ├── video_cap.cpp │ └── video_cap.hpp ├── tests/ │ ├── README.md │ ├── end_to_end_tests.py │ ├── reference/ │ │ ├── h264/ │ │ │ ├── frame_types.txt │ │ │ └── motion_vectors/ │ │ │ ├── mvs-0.npy │ │ │ ├── mvs-1.npy │ │ │ ├── mvs-10.npy │ │ │ ├── mvs-100.npy │ │ │ ├── mvs-101.npy │ │ │ ├── mvs-102.npy │ │ │ ├── mvs-103.npy │ │ │ ├── mvs-104.npy │ │ │ ├── mvs-105.npy │ │ │ ├── mvs-106.npy │ │ │ ├── mvs-107.npy │ │ │ ├── mvs-108.npy │ │ │ ├── mvs-109.npy │ │ │ ├── mvs-11.npy │ │ │ ├── mvs-110.npy │ │ │ ├── mvs-111.npy │ │ │ ├── mvs-112.npy │ │ │ ├── mvs-113.npy │ │ │ ├── mvs-114.npy │ │ │ ├── mvs-115.npy │ │ │ ├── mvs-116.npy │ │ │ ├── mvs-117.npy │ │ │ ├── mvs-118.npy │ │ │ ├── mvs-119.npy │ │ │ ├── mvs-12.npy │ │ │ ├── mvs-120.npy │ │ │ ├── mvs-121.npy │ │ │ ├── mvs-122.npy │ │ │ ├── mvs-123.npy │ │ │ ├── mvs-124.npy │ │ │ ├── mvs-125.npy │ │ │ ├── mvs-126.npy │ │ │ ├── mvs-127.npy │ │ │ ├── mvs-128.npy │ │ │ ├── mvs-129.npy │ │ │ ├── mvs-13.npy │ │ │ ├── mvs-130.npy │ │ │ ├── mvs-131.npy │ │ │ ├── mvs-132.npy │ │ │ ├── mvs-133.npy │ │ │ ├── mvs-134.npy │ │ │ ├── mvs-135.npy │ │ │ ├── mvs-136.npy │ │ │ ├── mvs-137.npy │ │ │ ├── mvs-138.npy │ │ │ ├── mvs-139.npy │ │ │ ├── mvs-14.npy │ │ │ ├── mvs-140.npy │ │ │ ├── mvs-141.npy │ │ │ ├── mvs-142.npy │ │ │ ├── mvs-143.npy │ │ │ ├── mvs-144.npy │ │ │ ├── mvs-145.npy │ │ │ ├── mvs-146.npy │ │ │ ├── mvs-147.npy │ │ │ ├── mvs-148.npy │ │ │ ├── mvs-149.npy │ │ │ ├── mvs-15.npy │ │ │ ├── mvs-150.npy │ │ │ ├── mvs-151.npy │ │ │ ├── mvs-152.npy │ │ │ ├── mvs-153.npy │ │ │ ├── mvs-154.npy │ │ │ ├── mvs-155.npy │ │ │ ├── mvs-156.npy │ │ │ ├── mvs-157.npy │ │ │ ├── mvs-158.npy │ │ │ ├── mvs-159.npy │ │ │ ├── mvs-16.npy │ │ │ ├── mvs-160.npy │ │ │ ├── mvs-161.npy │ │ │ ├── mvs-162.npy │ │ │ ├── mvs-163.npy │ │ │ ├── mvs-164.npy │ │ │ ├── mvs-165.npy │ │ │ ├── mvs-166.npy │ │ │ ├── mvs-167.npy │ │ │ ├── mvs-168.npy │ │ │ ├── mvs-169.npy │ │ │ ├── mvs-17.npy │ │ │ ├── mvs-170.npy │ │ │ ├── mvs-171.npy │ │ │ ├── mvs-172.npy │ │ │ ├── mvs-173.npy │ │ │ ├── mvs-174.npy │ │ │ ├── mvs-175.npy │ │ │ ├── mvs-176.npy │ │ │ ├── mvs-177.npy │ │ │ ├── mvs-178.npy │ │ │ ├── mvs-179.npy │ │ │ ├── mvs-18.npy │ │ │ ├── mvs-180.npy │ │ │ ├── mvs-181.npy │ │ │ ├── mvs-182.npy │ │ │ ├── mvs-183.npy │ │ │ ├── mvs-184.npy │ │ │ ├── mvs-185.npy │ │ │ ├── mvs-186.npy │ │ │ ├── mvs-187.npy │ │ │ ├── mvs-188.npy │ │ │ ├── mvs-189.npy │ │ │ ├── mvs-19.npy │ │ │ ├── mvs-190.npy │ │ │ ├── mvs-191.npy │ │ │ ├── mvs-192.npy │ │ │ ├── mvs-193.npy │ │ │ ├── mvs-194.npy │ │ │ ├── mvs-195.npy │ │ │ ├── mvs-196.npy │ │ │ ├── mvs-197.npy │ │ │ ├── mvs-198.npy │ │ │ ├── mvs-199.npy │ │ │ ├── mvs-2.npy │ │ │ ├── mvs-20.npy │ │ │ ├── mvs-200.npy │ │ │ ├── mvs-201.npy │ │ │ ├── mvs-202.npy │ │ │ ├── mvs-203.npy │ │ │ ├── mvs-204.npy │ │ │ ├── mvs-205.npy │ │ │ ├── mvs-206.npy │ │ │ ├── mvs-207.npy │ │ │ ├── mvs-208.npy │ │ │ ├── mvs-209.npy │ │ │ ├── mvs-21.npy │ │ │ ├── mvs-210.npy │ │ │ ├── mvs-211.npy │ │ │ ├── mvs-212.npy │ │ │ ├── mvs-213.npy │ │ │ ├── mvs-214.npy │ │ │ ├── mvs-215.npy │ │ │ ├── mvs-216.npy │ │ │ ├── mvs-217.npy │ │ │ ├── mvs-218.npy │ │ │ ├── mvs-219.npy │ │ │ ├── mvs-22.npy │ │ │ ├── mvs-220.npy │ │ │ ├── mvs-221.npy │ │ │ ├── mvs-222.npy │ │ │ ├── mvs-223.npy │ │ │ ├── mvs-224.npy │ │ │ ├── mvs-225.npy │ │ │ ├── mvs-226.npy │ │ │ ├── mvs-227.npy │ │ │ ├── mvs-228.npy │ │ │ ├── mvs-229.npy │ │ │ ├── mvs-23.npy │ │ │ ├── mvs-230.npy │ │ │ ├── mvs-231.npy │ │ │ ├── mvs-232.npy │ │ │ ├── mvs-233.npy │ │ │ ├── mvs-234.npy │ │ │ ├── mvs-235.npy │ │ │ ├── mvs-236.npy │ │ │ ├── mvs-237.npy │ │ │ ├── mvs-238.npy │ │ │ ├── mvs-239.npy │ │ │ ├── mvs-24.npy │ │ │ ├── mvs-240.npy │ │ │ ├── mvs-241.npy │ │ │ ├── mvs-242.npy │ │ │ ├── mvs-243.npy │ │ │ ├── mvs-244.npy │ │ │ ├── mvs-245.npy │ │ │ ├── mvs-246.npy │ │ │ ├── mvs-247.npy │ │ │ ├── mvs-248.npy │ │ │ ├── mvs-249.npy │ │ │ ├── mvs-25.npy │ │ │ ├── mvs-250.npy │ │ │ ├── mvs-251.npy │ │ │ ├── mvs-252.npy │ │ │ ├── mvs-253.npy │ │ │ ├── mvs-254.npy │ │ │ ├── mvs-255.npy │ │ │ ├── mvs-256.npy │ │ │ ├── mvs-257.npy │ │ │ ├── mvs-258.npy │ │ │ ├── mvs-259.npy │ │ │ ├── mvs-26.npy │ │ │ ├── mvs-260.npy │ │ │ ├── mvs-261.npy │ │ │ ├── mvs-262.npy │ │ │ ├── mvs-263.npy │ │ │ ├── mvs-264.npy │ │ │ ├── mvs-265.npy │ │ │ ├── mvs-266.npy │ │ │ ├── mvs-267.npy │ │ │ ├── mvs-268.npy │ │ │ ├── mvs-269.npy │ │ │ ├── mvs-27.npy │ │ │ ├── mvs-270.npy │ │ │ ├── mvs-271.npy │ │ │ ├── mvs-272.npy │ │ │ ├── mvs-273.npy │ │ │ ├── mvs-274.npy │ │ │ ├── mvs-275.npy │ │ │ ├── mvs-276.npy │ │ │ ├── mvs-277.npy │ │ │ ├── mvs-278.npy │ │ │ ├── mvs-279.npy │ │ │ ├── mvs-28.npy │ │ │ ├── mvs-280.npy │ │ │ ├── mvs-281.npy │ │ │ ├── mvs-282.npy │ │ │ ├── mvs-283.npy │ │ │ ├── mvs-284.npy │ │ │ ├── mvs-285.npy │ │ │ ├── mvs-286.npy │ │ │ ├── mvs-287.npy │ │ │ ├── mvs-288.npy │ │ │ ├── mvs-289.npy │ │ │ ├── mvs-29.npy │ │ │ ├── mvs-290.npy │ │ │ ├── mvs-291.npy │ │ │ ├── mvs-292.npy │ │ │ ├── mvs-293.npy │ │ │ ├── mvs-294.npy │ │ │ ├── mvs-295.npy │ │ │ ├── mvs-296.npy │ │ │ ├── mvs-297.npy │ │ │ ├── mvs-298.npy │ │ │ ├── mvs-299.npy │ │ │ ├── mvs-3.npy │ │ │ ├── mvs-30.npy │ │ │ ├── mvs-300.npy │ │ │ ├── mvs-301.npy │ │ │ ├── mvs-302.npy │ │ │ ├── mvs-303.npy │ │ │ ├── mvs-304.npy │ │ │ ├── mvs-305.npy │ │ │ ├── mvs-306.npy │ │ │ ├── mvs-307.npy │ │ │ ├── mvs-308.npy │ │ │ ├── mvs-309.npy │ │ │ ├── mvs-31.npy │ │ │ ├── mvs-310.npy │ │ │ ├── mvs-311.npy │ │ │ ├── mvs-312.npy │ │ │ ├── mvs-313.npy │ │ │ ├── mvs-314.npy │ │ │ ├── mvs-315.npy │ │ │ ├── mvs-316.npy │ │ │ ├── mvs-317.npy │ │ │ ├── mvs-318.npy │ │ │ ├── mvs-319.npy │ │ │ ├── mvs-32.npy │ │ │ ├── mvs-320.npy │ │ │ ├── mvs-321.npy │ │ │ ├── mvs-322.npy │ │ │ ├── mvs-323.npy │ │ │ ├── mvs-324.npy │ │ │ ├── mvs-325.npy │ │ │ ├── mvs-326.npy │ │ │ ├── mvs-327.npy │ │ │ ├── mvs-328.npy │ │ │ ├── mvs-329.npy │ │ │ ├── mvs-33.npy │ │ │ ├── mvs-330.npy │ │ │ ├── mvs-331.npy │ │ │ ├── mvs-332.npy │ │ │ ├── mvs-333.npy │ │ │ ├── mvs-334.npy │ │ │ ├── mvs-335.npy │ │ │ ├── mvs-336.npy │ │ │ ├── mvs-34.npy │ │ │ ├── mvs-35.npy │ │ │ ├── mvs-36.npy │ │ │ ├── mvs-37.npy │ │ │ ├── mvs-38.npy │ │ │ ├── mvs-39.npy │ │ │ ├── mvs-4.npy │ │ │ ├── mvs-40.npy │ │ │ ├── mvs-41.npy │ │ │ ├── mvs-42.npy │ │ │ ├── mvs-43.npy │ │ │ ├── mvs-44.npy │ │ │ ├── mvs-45.npy │ │ │ ├── mvs-46.npy │ │ │ ├── mvs-47.npy │ │ │ ├── mvs-48.npy │ │ │ ├── mvs-49.npy │ │ │ ├── mvs-5.npy │ │ │ ├── mvs-50.npy │ │ │ ├── mvs-51.npy │ │ │ ├── mvs-52.npy │ │ │ ├── mvs-53.npy │ │ │ ├── mvs-54.npy │ │ │ ├── mvs-55.npy │ │ │ ├── mvs-56.npy │ │ │ ├── mvs-57.npy │ │ │ ├── mvs-58.npy │ │ │ ├── mvs-59.npy │ │ │ ├── mvs-6.npy │ │ │ ├── mvs-60.npy │ │ │ ├── mvs-61.npy │ │ │ ├── mvs-62.npy │ │ │ ├── mvs-63.npy │ │ │ ├── mvs-64.npy │ │ │ ├── mvs-65.npy │ │ │ ├── mvs-66.npy │ │ │ ├── mvs-67.npy │ │ │ ├── mvs-68.npy │ │ │ ├── mvs-69.npy │ │ │ ├── mvs-7.npy │ │ │ ├── mvs-70.npy │ │ │ ├── mvs-71.npy │ │ │ ├── mvs-72.npy │ │ │ ├── mvs-73.npy │ │ │ ├── mvs-74.npy │ │ │ ├── mvs-75.npy │ │ │ ├── mvs-76.npy │ │ │ ├── mvs-77.npy │ │ │ ├── mvs-78.npy │ │ │ ├── mvs-79.npy │ │ │ ├── mvs-8.npy │ │ │ ├── mvs-80.npy │ │ │ ├── mvs-81.npy │ │ │ ├── mvs-82.npy │ │ │ ├── mvs-83.npy │ │ │ ├── mvs-84.npy │ │ │ ├── mvs-85.npy │ │ │ ├── mvs-86.npy │ │ │ ├── mvs-87.npy │ │ │ ├── mvs-88.npy │ │ │ ├── mvs-89.npy │ │ │ ├── mvs-9.npy │ │ │ ├── mvs-90.npy │ │ │ ├── mvs-91.npy │ │ │ ├── mvs-92.npy │ │ │ ├── mvs-93.npy │ │ │ ├── mvs-94.npy │ │ │ ├── mvs-95.npy │ │ │ ├── mvs-96.npy │ │ │ ├── mvs-97.npy │ │ │ ├── mvs-98.npy │ │ │ └── mvs-99.npy │ │ ├── mpeg4_part2/ │ │ │ ├── frame_types.txt │ │ │ └── motion_vectors/ │ │ │ ├── mvs-0.npy │ │ │ ├── mvs-1.npy │ │ │ ├── mvs-10.npy │ │ │ ├── mvs-100.npy │ │ │ ├── mvs-101.npy │ │ │ ├── mvs-102.npy │ │ │ ├── mvs-103.npy │ │ │ ├── mvs-104.npy │ │ │ ├── mvs-105.npy │ │ │ ├── mvs-106.npy │ │ │ ├── mvs-107.npy │ │ │ ├── mvs-108.npy │ │ │ ├── mvs-109.npy │ │ │ ├── mvs-11.npy │ │ │ ├── mvs-110.npy │ │ │ ├── mvs-111.npy │ │ │ ├── mvs-112.npy │ │ │ ├── mvs-113.npy │ │ │ ├── mvs-114.npy │ │ │ ├── mvs-115.npy │ │ │ ├── mvs-116.npy │ │ │ ├── mvs-117.npy │ │ │ ├── mvs-118.npy │ │ │ ├── mvs-119.npy │ │ │ ├── mvs-12.npy │ │ │ ├── mvs-120.npy │ │ │ ├── mvs-121.npy │ │ │ ├── mvs-122.npy │ │ │ ├── mvs-123.npy │ │ │ ├── mvs-124.npy │ │ │ ├── mvs-125.npy │ │ │ ├── mvs-126.npy │ │ │ ├── mvs-127.npy │ │ │ ├── mvs-128.npy │ │ │ ├── mvs-129.npy │ │ │ ├── mvs-13.npy │ │ │ ├── mvs-130.npy │ │ │ ├── mvs-131.npy │ │ │ ├── mvs-132.npy │ │ │ ├── mvs-133.npy │ │ │ ├── mvs-134.npy │ │ │ ├── mvs-135.npy │ │ │ ├── mvs-136.npy │ │ │ ├── mvs-137.npy │ │ │ ├── mvs-138.npy │ │ │ ├── mvs-139.npy │ │ │ ├── mvs-14.npy │ │ │ ├── mvs-140.npy │ │ │ ├── mvs-141.npy │ │ │ ├── mvs-142.npy │ │ │ ├── mvs-143.npy │ │ │ ├── mvs-144.npy │ │ │ ├── mvs-145.npy │ │ │ ├── mvs-146.npy │ │ │ ├── mvs-147.npy │ │ │ ├── mvs-148.npy │ │ │ ├── mvs-149.npy │ │ │ ├── mvs-15.npy │ │ │ ├── mvs-150.npy │ │ │ ├── mvs-151.npy │ │ │ ├── mvs-152.npy │ │ │ ├── mvs-153.npy │ │ │ ├── mvs-154.npy │ │ │ ├── mvs-155.npy │ │ │ ├── mvs-156.npy │ │ │ ├── mvs-157.npy │ │ │ ├── mvs-158.npy │ │ │ ├── mvs-159.npy │ │ │ ├── mvs-16.npy │ │ │ ├── mvs-160.npy │ │ │ ├── mvs-161.npy │ │ │ ├── mvs-162.npy │ │ │ ├── mvs-163.npy │ │ │ ├── mvs-164.npy │ │ │ ├── mvs-165.npy │ │ │ ├── mvs-166.npy │ │ │ ├── mvs-167.npy │ │ │ ├── mvs-168.npy │ │ │ ├── mvs-169.npy │ │ │ ├── mvs-17.npy │ │ │ ├── mvs-170.npy │ │ │ ├── mvs-171.npy │ │ │ ├── mvs-172.npy │ │ │ ├── mvs-173.npy │ │ │ ├── mvs-174.npy │ │ │ ├── mvs-175.npy │ │ │ ├── mvs-176.npy │ │ │ ├── mvs-177.npy │ │ │ ├── mvs-178.npy │ │ │ ├── mvs-179.npy │ │ │ ├── mvs-18.npy │ │ │ ├── mvs-180.npy │ │ │ ├── mvs-181.npy │ │ │ ├── mvs-182.npy │ │ │ ├── mvs-183.npy │ │ │ ├── mvs-184.npy │ │ │ ├── mvs-185.npy │ │ │ ├── mvs-186.npy │ │ │ ├── mvs-187.npy │ │ │ ├── mvs-188.npy │ │ │ ├── mvs-189.npy │ │ │ ├── mvs-19.npy │ │ │ ├── mvs-190.npy │ │ │ ├── mvs-191.npy │ │ │ ├── mvs-192.npy │ │ │ ├── mvs-193.npy │ │ │ ├── mvs-194.npy │ │ │ ├── mvs-195.npy │ │ │ ├── mvs-196.npy │ │ │ ├── mvs-197.npy │ │ │ ├── mvs-198.npy │ │ │ ├── mvs-199.npy │ │ │ ├── mvs-2.npy │ │ │ ├── mvs-20.npy │ │ │ ├── mvs-200.npy │ │ │ ├── mvs-201.npy │ │ │ ├── mvs-202.npy │ │ │ ├── mvs-203.npy │ │ │ ├── mvs-204.npy │ │ │ ├── mvs-205.npy │ │ │ ├── mvs-206.npy │ │ │ ├── mvs-207.npy │ │ │ ├── mvs-208.npy │ │ │ ├── mvs-209.npy │ │ │ ├── mvs-21.npy │ │ │ ├── mvs-210.npy │ │ │ ├── mvs-211.npy │ │ │ ├── mvs-212.npy │ │ │ ├── mvs-213.npy │ │ │ ├── mvs-214.npy │ │ │ ├── mvs-215.npy │ │ │ ├── mvs-216.npy │ │ │ ├── mvs-217.npy │ │ │ ├── mvs-218.npy │ │ │ ├── mvs-219.npy │ │ │ ├── mvs-22.npy │ │ │ ├── mvs-220.npy │ │ │ ├── mvs-221.npy │ │ │ ├── mvs-222.npy │ │ │ ├── mvs-223.npy │ │ │ ├── mvs-224.npy │ │ │ ├── mvs-225.npy │ │ │ ├── mvs-226.npy │ │ │ ├── mvs-227.npy │ │ │ ├── mvs-228.npy │ │ │ ├── mvs-229.npy │ │ │ ├── mvs-23.npy │ │ │ ├── mvs-230.npy │ │ │ ├── mvs-231.npy │ │ │ ├── mvs-232.npy │ │ │ ├── mvs-233.npy │ │ │ ├── mvs-234.npy │ │ │ ├── mvs-235.npy │ │ │ ├── mvs-236.npy │ │ │ ├── mvs-237.npy │ │ │ ├── mvs-238.npy │ │ │ ├── mvs-239.npy │ │ │ ├── mvs-24.npy │ │ │ ├── mvs-240.npy │ │ │ ├── mvs-241.npy │ │ │ ├── mvs-242.npy │ │ │ ├── mvs-243.npy │ │ │ ├── mvs-244.npy │ │ │ ├── mvs-245.npy │ │ │ ├── mvs-246.npy │ │ │ ├── mvs-247.npy │ │ │ ├── mvs-248.npy │ │ │ ├── mvs-249.npy │ │ │ ├── mvs-25.npy │ │ │ ├── mvs-250.npy │ │ │ ├── mvs-251.npy │ │ │ ├── mvs-252.npy │ │ │ ├── mvs-253.npy │ │ │ ├── mvs-254.npy │ │ │ ├── mvs-255.npy │ │ │ ├── mvs-256.npy │ │ │ ├── mvs-257.npy │ │ │ ├── mvs-258.npy │ │ │ ├── mvs-259.npy │ │ │ ├── mvs-26.npy │ │ │ ├── mvs-260.npy │ │ │ ├── mvs-261.npy │ │ │ ├── mvs-262.npy │ │ │ ├── mvs-263.npy │ │ │ ├── mvs-264.npy │ │ │ ├── mvs-265.npy │ │ │ ├── mvs-266.npy │ │ │ ├── mvs-267.npy │ │ │ ├── mvs-268.npy │ │ │ ├── mvs-269.npy │ │ │ ├── mvs-27.npy │ │ │ ├── mvs-270.npy │ │ │ ├── mvs-271.npy │ │ │ ├── mvs-272.npy │ │ │ ├── mvs-273.npy │ │ │ ├── mvs-274.npy │ │ │ ├── mvs-275.npy │ │ │ ├── mvs-276.npy │ │ │ ├── mvs-277.npy │ │ │ ├── mvs-278.npy │ │ │ ├── mvs-279.npy │ │ │ ├── mvs-28.npy │ │ │ ├── mvs-280.npy │ │ │ ├── mvs-281.npy │ │ │ ├── mvs-282.npy │ │ │ ├── mvs-283.npy │ │ │ ├── mvs-284.npy │ │ │ ├── mvs-285.npy │ │ │ ├── mvs-286.npy │ │ │ ├── mvs-287.npy │ │ │ ├── mvs-288.npy │ │ │ ├── mvs-289.npy │ │ │ ├── mvs-29.npy │ │ │ ├── mvs-290.npy │ │ │ ├── mvs-291.npy │ │ │ ├── mvs-292.npy │ │ │ ├── mvs-293.npy │ │ │ ├── mvs-294.npy │ │ │ ├── mvs-295.npy │ │ │ ├── mvs-296.npy │ │ │ ├── mvs-297.npy │ │ │ ├── mvs-298.npy │ │ │ ├── mvs-299.npy │ │ │ ├── mvs-3.npy │ │ │ ├── mvs-30.npy │ │ │ ├── mvs-300.npy │ │ │ ├── mvs-301.npy │ │ │ ├── mvs-302.npy │ │ │ ├── mvs-303.npy │ │ │ ├── mvs-304.npy │ │ │ ├── mvs-305.npy │ │ │ ├── mvs-306.npy │ │ │ ├── mvs-307.npy │ │ │ ├── mvs-308.npy │ │ │ ├── mvs-309.npy │ │ │ ├── mvs-31.npy │ │ │ ├── mvs-310.npy │ │ │ ├── mvs-311.npy │ │ │ ├── mvs-312.npy │ │ │ ├── mvs-313.npy │ │ │ ├── mvs-314.npy │ │ │ ├── mvs-315.npy │ │ │ ├── mvs-316.npy │ │ │ ├── mvs-317.npy │ │ │ ├── mvs-318.npy │ │ │ ├── mvs-319.npy │ │ │ ├── mvs-32.npy │ │ │ ├── mvs-320.npy │ │ │ ├── mvs-321.npy │ │ │ ├── mvs-322.npy │ │ │ ├── mvs-323.npy │ │ │ ├── mvs-324.npy │ │ │ ├── mvs-325.npy │ │ │ ├── mvs-326.npy │ │ │ ├── mvs-327.npy │ │ │ ├── mvs-328.npy │ │ │ ├── mvs-329.npy │ │ │ ├── mvs-33.npy │ │ │ ├── mvs-330.npy │ │ │ ├── mvs-331.npy │ │ │ ├── mvs-332.npy │ │ │ ├── mvs-333.npy │ │ │ ├── mvs-334.npy │ │ │ ├── mvs-335.npy │ │ │ ├── mvs-336.npy │ │ │ ├── mvs-34.npy │ │ │ ├── mvs-35.npy │ │ │ ├── mvs-36.npy │ │ │ ├── mvs-37.npy │ │ │ ├── mvs-38.npy │ │ │ ├── mvs-39.npy │ │ │ ├── mvs-4.npy │ │ │ ├── mvs-40.npy │ │ │ ├── mvs-41.npy │ │ │ ├── mvs-42.npy │ │ │ ├── mvs-43.npy │ │ │ ├── mvs-44.npy │ │ │ ├── mvs-45.npy │ │ │ ├── mvs-46.npy │ │ │ ├── mvs-47.npy │ │ │ ├── mvs-48.npy │ │ │ ├── mvs-49.npy │ │ │ ├── mvs-5.npy │ │ │ ├── mvs-50.npy │ │ │ ├── mvs-51.npy │ │ │ ├── mvs-52.npy │ │ │ ├── mvs-53.npy │ │ │ ├── mvs-54.npy │ │ │ ├── mvs-55.npy │ │ │ ├── mvs-56.npy │ │ │ ├── mvs-57.npy │ │ │ ├── mvs-58.npy │ │ │ ├── mvs-59.npy │ │ │ ├── mvs-6.npy │ │ │ ├── mvs-60.npy │ │ │ ├── mvs-61.npy │ │ │ ├── mvs-62.npy │ │ │ ├── mvs-63.npy │ │ │ ├── mvs-64.npy │ │ │ ├── mvs-65.npy │ │ │ ├── mvs-66.npy │ │ │ ├── mvs-67.npy │ │ │ ├── mvs-68.npy │ │ │ ├── mvs-69.npy │ │ │ ├── mvs-7.npy │ │ │ ├── mvs-70.npy │ │ │ ├── mvs-71.npy │ │ │ ├── mvs-72.npy │ │ │ ├── mvs-73.npy │ │ │ ├── mvs-74.npy │ │ │ ├── mvs-75.npy │ │ │ ├── mvs-76.npy │ │ │ ├── mvs-77.npy │ │ │ ├── mvs-78.npy │ │ │ ├── mvs-79.npy │ │ │ ├── mvs-8.npy │ │ │ ├── mvs-80.npy │ │ │ ├── mvs-81.npy │ │ │ ├── mvs-82.npy │ │ │ ├── mvs-83.npy │ │ │ ├── mvs-84.npy │ │ │ ├── mvs-85.npy │ │ │ ├── mvs-86.npy │ │ │ ├── mvs-87.npy │ │ │ ├── mvs-88.npy │ │ │ ├── mvs-89.npy │ │ │ ├── mvs-9.npy │ │ │ ├── mvs-90.npy │ │ │ ├── mvs-91.npy │ │ │ ├── mvs-92.npy │ │ │ ├── mvs-93.npy │ │ │ ├── mvs-94.npy │ │ │ ├── mvs-95.npy │ │ │ ├── mvs-96.npy │ │ │ ├── mvs-97.npy │ │ │ ├── mvs-98.npy │ │ │ └── mvs-99.npy │ │ └── rtsp/ │ │ ├── frame_types.txt │ │ └── motion_vectors/ │ │ ├── mvs-0.npy │ │ ├── mvs-1.npy │ │ ├── mvs-10.npy │ │ ├── mvs-100.npy │ │ ├── mvs-101.npy │ │ ├── mvs-102.npy │ │ ├── mvs-103.npy │ │ ├── mvs-104.npy │ │ ├── mvs-105.npy │ │ ├── mvs-106.npy │ │ ├── mvs-107.npy │ │ ├── mvs-108.npy │ │ ├── mvs-109.npy │ │ ├── mvs-11.npy │ │ ├── mvs-110.npy │ │ ├── mvs-111.npy │ │ ├── mvs-112.npy │ │ ├── mvs-113.npy │ │ ├── mvs-114.npy │ │ ├── mvs-115.npy │ │ ├── mvs-116.npy │ │ ├── mvs-117.npy │ │ ├── mvs-118.npy │ │ ├── mvs-119.npy │ │ ├── mvs-12.npy │ │ ├── mvs-120.npy │ │ ├── mvs-121.npy │ │ ├── mvs-122.npy │ │ ├── mvs-123.npy │ │ ├── mvs-124.npy │ │ ├── mvs-125.npy │ │ ├── mvs-126.npy │ │ ├── mvs-127.npy │ │ ├── mvs-128.npy │ │ ├── mvs-129.npy │ │ ├── mvs-13.npy │ │ ├── mvs-130.npy │ │ ├── mvs-131.npy │ │ ├── mvs-132.npy │ │ ├── mvs-133.npy │ │ ├── mvs-134.npy │ │ ├── mvs-135.npy │ │ ├── mvs-136.npy │ │ ├── mvs-137.npy │ │ ├── mvs-138.npy │ │ ├── mvs-139.npy │ │ ├── mvs-14.npy │ │ ├── mvs-140.npy │ │ ├── mvs-141.npy │ │ ├── mvs-142.npy │ │ ├── mvs-143.npy │ │ ├── mvs-144.npy │ │ ├── mvs-145.npy │ │ ├── mvs-146.npy │ │ ├── mvs-147.npy │ │ ├── mvs-148.npy │ │ ├── mvs-149.npy │ │ ├── mvs-15.npy │ │ ├── mvs-150.npy │ │ ├── mvs-151.npy │ │ ├── mvs-152.npy │ │ ├── mvs-153.npy │ │ ├── mvs-154.npy │ │ ├── mvs-155.npy │ │ ├── mvs-156.npy │ │ ├── mvs-157.npy │ │ ├── mvs-158.npy │ │ ├── mvs-159.npy │ │ ├── mvs-16.npy │ │ ├── mvs-160.npy │ │ ├── mvs-161.npy │ │ ├── mvs-162.npy │ │ ├── mvs-163.npy │ │ ├── mvs-164.npy │ │ ├── mvs-165.npy │ │ ├── mvs-166.npy │ │ ├── mvs-167.npy │ │ ├── mvs-168.npy │ │ ├── mvs-169.npy │ │ ├── mvs-17.npy │ │ ├── mvs-170.npy │ │ ├── mvs-171.npy │ │ ├── mvs-172.npy │ │ ├── mvs-173.npy │ │ ├── mvs-174.npy │ │ ├── mvs-175.npy │ │ ├── mvs-176.npy │ │ ├── mvs-177.npy │ │ ├── mvs-178.npy │ │ ├── mvs-179.npy │ │ ├── mvs-18.npy │ │ ├── mvs-180.npy │ │ ├── mvs-181.npy │ │ ├── mvs-182.npy │ │ ├── mvs-183.npy │ │ ├── mvs-184.npy │ │ ├── mvs-185.npy │ │ ├── mvs-186.npy │ │ ├── mvs-187.npy │ │ ├── mvs-188.npy │ │ ├── mvs-189.npy │ │ ├── mvs-19.npy │ │ ├── mvs-190.npy │ │ ├── mvs-191.npy │ │ ├── mvs-192.npy │ │ ├── mvs-193.npy │ │ ├── mvs-194.npy │ │ ├── mvs-195.npy │ │ ├── mvs-196.npy │ │ ├── mvs-197.npy │ │ ├── mvs-198.npy │ │ ├── mvs-199.npy │ │ ├── mvs-2.npy │ │ ├── mvs-20.npy │ │ ├── mvs-200.npy │ │ ├── mvs-201.npy │ │ ├── mvs-202.npy │ │ ├── mvs-203.npy │ │ ├── mvs-204.npy │ │ ├── mvs-205.npy │ │ ├── mvs-206.npy │ │ ├── mvs-207.npy │ │ ├── mvs-208.npy │ │ ├── mvs-209.npy │ │ ├── mvs-21.npy │ │ ├── mvs-210.npy │ │ ├── mvs-211.npy │ │ ├── mvs-212.npy │ │ ├── mvs-213.npy │ │ ├── mvs-214.npy │ │ ├── mvs-215.npy │ │ ├── mvs-216.npy │ │ ├── mvs-217.npy │ │ ├── mvs-218.npy │ │ ├── mvs-219.npy │ │ ├── mvs-22.npy │ │ ├── mvs-220.npy │ │ ├── mvs-221.npy │ │ ├── mvs-222.npy │ │ ├── mvs-223.npy │ │ ├── mvs-224.npy │ │ ├── mvs-225.npy │ │ ├── mvs-226.npy │ │ ├── mvs-227.npy │ │ ├── mvs-228.npy │ │ ├── mvs-229.npy │ │ ├── mvs-23.npy │ │ ├── mvs-230.npy │ │ ├── mvs-231.npy │ │ ├── mvs-232.npy │ │ ├── mvs-233.npy │ │ ├── mvs-234.npy │ │ ├── mvs-235.npy │ │ ├── mvs-236.npy │ │ ├── mvs-237.npy │ │ ├── mvs-238.npy │ │ ├── mvs-239.npy │ │ ├── mvs-24.npy │ │ ├── mvs-240.npy │ │ ├── mvs-241.npy │ │ ├── mvs-242.npy │ │ ├── mvs-243.npy │ │ ├── mvs-244.npy │ │ ├── mvs-245.npy │ │ ├── mvs-246.npy │ │ ├── mvs-247.npy │ │ ├── mvs-248.npy │ │ ├── mvs-249.npy │ │ ├── mvs-25.npy │ │ ├── mvs-250.npy │ │ ├── mvs-251.npy │ │ ├── mvs-252.npy │ │ ├── mvs-253.npy │ │ ├── mvs-254.npy │ │ ├── mvs-255.npy │ │ ├── mvs-256.npy │ │ ├── mvs-257.npy │ │ ├── mvs-258.npy │ │ ├── mvs-259.npy │ │ ├── mvs-26.npy │ │ ├── mvs-260.npy │ │ ├── mvs-261.npy │ │ ├── mvs-262.npy │ │ ├── mvs-263.npy │ │ ├── mvs-264.npy │ │ ├── mvs-265.npy │ │ ├── mvs-266.npy │ │ ├── mvs-267.npy │ │ ├── mvs-268.npy │ │ ├── mvs-269.npy │ │ ├── mvs-27.npy │ │ ├── mvs-270.npy │ │ ├── mvs-271.npy │ │ ├── mvs-272.npy │ │ ├── mvs-273.npy │ │ ├── mvs-274.npy │ │ ├── mvs-275.npy │ │ ├── mvs-276.npy │ │ ├── mvs-277.npy │ │ ├── mvs-278.npy │ │ ├── mvs-279.npy │ │ ├── mvs-28.npy │ │ ├── mvs-280.npy │ │ ├── mvs-281.npy │ │ ├── mvs-282.npy │ │ ├── mvs-283.npy │ │ ├── mvs-284.npy │ │ ├── mvs-285.npy │ │ ├── mvs-286.npy │ │ ├── mvs-287.npy │ │ ├── mvs-288.npy │ │ ├── mvs-289.npy │ │ ├── mvs-29.npy │ │ ├── mvs-290.npy │ │ ├── mvs-291.npy │ │ ├── mvs-292.npy │ │ ├── mvs-293.npy │ │ ├── mvs-294.npy │ │ ├── mvs-295.npy │ │ ├── mvs-296.npy │ │ ├── mvs-297.npy │ │ ├── mvs-298.npy │ │ ├── mvs-299.npy │ │ ├── mvs-3.npy │ │ ├── mvs-30.npy │ │ ├── mvs-300.npy │ │ ├── mvs-301.npy │ │ ├── mvs-302.npy │ │ ├── mvs-303.npy │ │ ├── mvs-304.npy │ │ ├── mvs-305.npy │ │ ├── mvs-306.npy │ │ ├── mvs-307.npy │ │ ├── mvs-308.npy │ │ ├── mvs-309.npy │ │ ├── mvs-31.npy │ │ ├── mvs-310.npy │ │ ├── mvs-311.npy │ │ ├── mvs-312.npy │ │ ├── mvs-313.npy │ │ ├── mvs-314.npy │ │ ├── mvs-315.npy │ │ ├── mvs-316.npy │ │ ├── mvs-317.npy │ │ ├── mvs-318.npy │ │ ├── mvs-319.npy │ │ ├── mvs-32.npy │ │ ├── mvs-320.npy │ │ ├── mvs-321.npy │ │ ├── mvs-322.npy │ │ ├── mvs-323.npy │ │ ├── mvs-324.npy │ │ ├── mvs-325.npy │ │ ├── mvs-326.npy │ │ ├── mvs-327.npy │ │ ├── mvs-328.npy │ │ ├── mvs-329.npy │ │ ├── mvs-33.npy │ │ ├── mvs-330.npy │ │ ├── mvs-331.npy │ │ ├── mvs-332.npy │ │ ├── mvs-333.npy │ │ ├── mvs-334.npy │ │ ├── mvs-335.npy │ │ ├── mvs-34.npy │ │ ├── mvs-35.npy │ │ ├── mvs-36.npy │ │ ├── mvs-37.npy │ │ ├── mvs-38.npy │ │ ├── mvs-39.npy │ │ ├── mvs-4.npy │ │ ├── mvs-40.npy │ │ ├── mvs-41.npy │ │ ├── mvs-42.npy │ │ ├── mvs-43.npy │ │ ├── mvs-44.npy │ │ ├── mvs-45.npy │ │ ├── mvs-46.npy │ │ ├── mvs-47.npy │ │ ├── mvs-48.npy │ │ ├── mvs-49.npy │ │ ├── mvs-5.npy │ │ ├── mvs-50.npy │ │ ├── mvs-51.npy │ │ ├── mvs-52.npy │ │ ├── mvs-53.npy │ │ ├── mvs-54.npy │ │ ├── mvs-55.npy │ │ ├── mvs-56.npy │ │ ├── mvs-57.npy │ │ ├── mvs-58.npy │ │ ├── mvs-59.npy │ │ ├── mvs-6.npy │ │ ├── mvs-60.npy │ │ ├── mvs-61.npy │ │ ├── mvs-62.npy │ │ ├── mvs-63.npy │ │ ├── mvs-64.npy │ │ ├── mvs-65.npy │ │ ├── mvs-66.npy │ │ ├── mvs-67.npy │ │ ├── mvs-68.npy │ │ ├── mvs-69.npy │ │ ├── mvs-7.npy │ │ ├── mvs-70.npy │ │ ├── mvs-71.npy │ │ ├── mvs-72.npy │ │ ├── mvs-73.npy │ │ ├── mvs-74.npy │ │ ├── mvs-75.npy │ │ ├── mvs-76.npy │ │ ├── mvs-77.npy │ │ ├── mvs-78.npy │ │ ├── mvs-79.npy │ │ ├── mvs-8.npy │ │ ├── mvs-80.npy │ │ ├── mvs-81.npy │ │ ├── mvs-82.npy │ │ ├── mvs-83.npy │ │ ├── mvs-84.npy │ │ ├── mvs-85.npy │ │ ├── mvs-86.npy │ │ ├── mvs-87.npy │ │ ├── mvs-88.npy │ │ ├── mvs-89.npy │ │ ├── mvs-9.npy │ │ ├── mvs-90.npy │ │ ├── mvs-91.npy │ │ ├── mvs-92.npy │ │ ├── mvs-93.npy │ │ ├── mvs-94.npy │ │ ├── mvs-95.npy │ │ ├── mvs-96.npy │ │ ├── mvs-97.npy │ │ ├── mvs-98.npy │ │ └── mvs-99.npy │ ├── tools/ │ │ └── live555MediaServer │ └── unit_tests.py └── vid_h264.264 ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/ci.yml ================================================ # On a push to any branch this workflow: # - builds the Docker image, # - build wheels for different Python versions, # - installs the wheel for each Python version and runs the unit tests # against newest and oldest versions of dependencies. # On manual dispatch this workflow: # - pushes the previously built Docker image to DockerHub with tag "dev". name: ci and release on: workflow_dispatch: pull_request: jobs: build_docker: name: Build Docker image runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to DockerHub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and export uses: docker/build-push-action@v6 with: context: . tags: mv-extractor:local outputs: type=docker,dest=/tmp/image.tar cache-from: type=registry,ref=lubo1994/mv-extractor:buildcache cache-to: type=registry,ref=lubo1994/mv-extractor:buildcache,mode=max - name: Upload Docker image as artifact uses: actions/upload-artifact@v4 with: name: mv-extractor-docker-image path: /tmp/image.tar test_docker: name: Run unit tests in Docker container (only for the Python version used in the Dockerfile command) runs-on: ubuntu-latest needs: - build_docker steps: - name: Checkout uses: actions/checkout@v4 - name: Download artifact containing Docker image uses: actions/download-artifact@v4 with: name: mv-extractor-docker-image path: /tmp - name: Load Docker image run: | docker load --input /tmp/image.tar - name: Run unit tests run: | docker run -v ${{ github.workspace }}:/home/video_cap \ mv-extractor:local \ /bin/bash -c ' \ yum install -y compat-openssl10 && \ python3.12 -m unittest discover -s tests -p "*tests.py" ' build_and_test_wheels: name: Build wheels for cp${{ matrix.python }}-${{ matrix.platform_id }} runs-on: ${{ matrix.os }} needs: - build_docker strategy: # Ensure that a wheel builder finishes even if another fails fail-fast: false matrix: include: - os: ubuntu-latest python: 39 bitness: 64 platform_id: manylinux_x86_64 manylinux_image: mv-extractor:local numpy_min_version: "numpy==1.19.3" opencv_min_version: "opencv-python==4.4.0.46" - os: ubuntu-latest python: 310 bitness: 64 platform_id: manylinux_x86_64 manylinux_image: mv-extractor:local numpy_min_version: "numpy==1.21.2" opencv_min_version: "opencv-python==4.5.4.60" - os: ubuntu-latest python: 311 bitness: 64 platform_id: manylinux_x86_64 manylinux_image: mv-extractor:local numpy_min_version: "numpy==1.23.3" opencv_min_version: "opencv-python==4.7.0.72" - os: ubuntu-latest python: 312 bitness: 64 platform_id: manylinux_x86_64 manylinux_image: mv-extractor:local numpy_min_version: "numpy==1.26.0" opencv_min_version: "opencv-python==4.9.0.80" - os: ubuntu-latest python: 313 bitness: 64 platform_id: manylinux_x86_64 manylinux_image: mv-extractor:local numpy_min_version: "numpy==2.2.6" opencv_min_version: "opencv-python==4.12.0.88" - os: ubuntu-latest python: 314 bitness: 64 platform_id: manylinux_x86_64 manylinux_image: mv-extractor:local numpy_min_version: "numpy==2.2.6" opencv_min_version: "opencv-python==4.12.0.88" steps: - name: Checkout uses: actions/checkout@v4 - name: Download artifact containing Docker image uses: actions/download-artifact@v4 with: name: mv-extractor-docker-image path: /tmp - name: Load Docker image run: | docker load --input /tmp/image.tar - name: Build and test wheels uses: pypa/cibuildwheel@v3.2.1 env: CIBW_PLATFORM: linux CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} # Disable building PyPy wheels on all platforms CIBW_SKIP: pp* CIBW_ARCHS: x86_64 CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} #CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} CIBW_BUILD_FRONTEND: build CIBW_TEST_COMMAND: | echo "Running unit tests" && \ yum install -y compat-openssl10 && \ PROJECT_ROOT={project} python3 -m unittest discover -s {project}/tests -p "*tests.py" && \ echo "Running unit tests against oldest supported versions of dependencies" && \ python3 -m pip install ${{ matrix.numpy_min_version }} ${{ matrix.opencv_min_version }} && \ PROJECT_ROOT={project} python3 -m unittest discover -s {project}/tests -p "*tests.py" CIBW_BUILD_VERBOSITY: 1 - uses: actions/upload-artifact@v4 with: name: python-wheel-${{ matrix.python }} path: ./wheelhouse/*.whl push_docker: name: Push Docker image to DockerHub if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest needs: - build_docker - test_docker - build_and_test_wheels steps: - name: Download artifact containing Docker image uses: actions/download-artifact@v4 with: name: mv-extractor-docker-image path: /tmp - name: Login to DockerHub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and push Docker image run: | docker load --input /tmp/image.tar docker tag mv-extractor:local lubo1994/mv-extractor:dev docker push lubo1994/mv-extractor:dev ================================================ FILE: .gitignore ================================================ /build/ /dist/ /wheelhouse/ *.egg-info *.egg .eggs __pycache__/ /venv3.*/ env/ out-*/ *.tar a.out *.so ================================================ FILE: Dockerfile ================================================ FROM quay.io/pypa/manylinux_2_28_x86_64 AS builder # Install build tools RUN yum update -y && \ yum install -y \ wget \ unzip \ git \ make \ cmake \ gcc-toolset-10 \ gcc-c++ \ pkgconfig \ libtool && \ yum clean all # Activate specific version of gcc toolset (newer versions of gcc fail to build old versions of ffmpeg) ENV PATH="/opt/rh/gcc-toolset-10/root/usr/bin:$PATH" ENV LD_LIBRARY_PATH="/opt/rh/gcc-toolset-10/root/usr/lib64:$LD_LIBRARY_PATH" # Install OpenCV ARG OPENCV_VERSION="4.12.0" WORKDIR /opt RUN wget -O opencv.zip https://github.com/opencv/opencv/archive/"$OPENCV_VERSION".zip && \ unzip opencv.zip && \ mv opencv-"$OPENCV_VERSION" opencv && \ mkdir opencv/build && \ cd opencv/build && \ cmake \ -D CMAKE_BUILD_TYPE=RELEASE \ -D OPENCV_GENERATE_PKGCONFIG=YES \ -D CMAKE_INSTALL_PREFIX=/usr/local \ -D OPENCV_ENABLE_NONFREE=OFF \ -D BUILD_LIST=core,imgproc \ .. && \ make -j $(nproc) && \ make install && \ ldconfig && \ rm -rf ../../opencv.zip && \ rm -rf ../../opencv # Install FFMPEG WORKDIR /opt/ffmpeg_sources RUN curl -O -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && \ tar xjvf nasm-2.15.05.tar.bz2 && \ cd nasm-2.15.05 && \ ./autogen.sh && \ ./configure --disable-shared --enable-static && \ make -j $(nproc) && \ make install && \ rm -rf ../nasm-2.15.05.tar.bz2 && \ rm -rf ../nasm-2.15.05 WORKDIR /opt/ffmpeg_sources RUN curl -O -L https://www.tortall.net/projects/yasm/releases/yasm-1.3.0.tar.gz && \ tar xzvf yasm-1.3.0.tar.gz && \ cd yasm-1.3.0 && \ ./configure --disable-shared --enable-static && \ make -j $(nproc) && \ make install && \ rm -rf ../yasm-1.3.0.tar.gz && \ rm -rf ../yasm-1.3.0 WORKDIR /opt/ffmpeg_sources RUN git clone --branch stable --depth 1 https://code.videolan.org/videolan/x264.git && \ cd x264 && \ ./configure --disable-shared --enable-static --enable-pic && \ make -j $(nproc) && \ make install && \ rm -rf ../x264 ARG FFMPEG_VERSION="4.1.3" WORKDIR /opt/ffmpeg_sources RUN wget -O ffmpeg-snapshot.tar.bz2 https://ffmpeg.org/releases/ffmpeg-"$FFMPEG_VERSION".tar.bz2 && \ mkdir -p ffmpeg && \ tar xjvf ffmpeg-snapshot.tar.bz2 -C ffmpeg --strip-components=1 && \ rm -rf ffmpeg-snapshot.tar.bz2 WORKDIR /opt/ffmpeg_sources/ffmpeg RUN ./configure \ --pkg-config-flags="--static" \ --extra-cflags="-I/usr/local/include" \ --extra-ldflags="-L/usr/local/lib" \ --extra-libs=-lpthread \ --extra-libs=-lm \ --enable-static \ --disable-shared \ --enable-gpl \ --enable-libx264 \ --enable-nonfree \ --enable-pic && \ make -j $(nproc) && \ make install && \ rm -rf ../ffmpeg FROM quay.io/pypa/manylinux_2_28_x86_64 # copy libraries WORKDIR /usr/local/lib COPY --from=builder /usr/local/lib . WORKDIR /usr/local/lib64 COPY --from=builder /usr/local/lib64 . WORKDIR /usr/local/include COPY --from=builder /usr/local/include . WORKDIR /usr/local/lib COPY --from=builder /usr/local/lib . # Set environment variables ENV PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/usr/local/lib64/pkgconfig" ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib64" WORKDIR /home/video_cap COPY pyproject.toml /home/video_cap/ COPY setup.py /home/video_cap/ COPY src /home/video_cap/src/ COPY README.md /home/video_cap/ # Install Python package RUN python3.12 -m pip install . # Location of the "extract_mvs" script ENV PATH="$PATH:/opt/python/cp312-cp312/bin" CMD ["sh", "-c", "tail -f /dev/null"] ================================================ FILE: LICENSE ================================================ The MIT License Copyright (c) 2010-2024 Lukas Bommes Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: MANIFEST.in ================================================ recursive-include ffmpeg_patch * recursive-include src * recursive-include tests * include LICENSE include pyproject.toml include extract_mvs.py include setup.py include vid_h264.mp4 include vid_mpeg4_part2.mp4 include vid_h264.264 ================================================ FILE: README.md ================================================

mvextractor
Motion Vector Extractor

This tool extracts motion vectors, frames, and frame types from H.264 and MPEG-4 Part 2 encoded videos. A replacement for OpenCV's [VideoCapture](https://docs.opencv.org/4.1.0/d8/dfe/classcv_1_1VideoCapture.html) that returns for each frame: - Frame type (I, P, or B) - motion vectors - Optional decoded frame as BGR image Frame decoding can be skipped for very fast motion vector extraction, ideal for, e.g., fast visual object tracking. Both a C++ and a Python API is provided. The image below shows a video frame with extracted motion vectors overlaid. ![motion_vector_demo_image](https://raw.githubusercontent.com/LukasBommes/mv-extractor/cb8e08f4c1e161d103d5382ded93134f26e96f05/mvs.png)
Note on Deprecation of Timestamp Extraction Versions 1.x of the motion vector extractor additionally returned the timestamps of video frames. For RTSP streams, the UTC wall time of when the sender transmitted a frame was returned (rather than the more easily retrievable reception timestamp). Since this feature required patching FFmpeg internals, it became difficult to maintain and prevented compatibility with newer versions of FFmpeg. As a result, timestamp extraction was removed in the 2.0.0 release. If you rely on this feature, please use version **1.1.0**.
## News ### Recent Changes in Release 2.0.0 - New motion-vectors-only mode, in which frame decoding is skipped for better performance (thanks to [@microa](https://github.com/LukasBommes/mv-extractor/pull/78)) - Dropped extraction of timestamps as this feature was complex and difficult to maintain. Note the breaking API change to the `read` and `retrieve` methods of the `VideoCapture` class ```diff - ret, frame, motion_vectors, frame_type, timestamp = cap.read() + ret, frame, motion_vectors, frame_type = cap.read() ``` - Added support for Python 3.13 and 3.14 - Moved installation of FFMPEG and OpenCV from script files directly into Dockerfile - Improved quickstart section of the readme ## Quickstart ### Step 1: Install ```bash pip install motion-vector-extractor ``` Note, that we currently provide the package only for x86-64 linux, such as Ubuntu or Debian, and Python 3.9 to 3.14. If you are on a different platform, please use the Docker image as described [below](#installation-via-docker). ### Step 2: Extract Motion Vectors You can follow along the examples below using the example video [`vid_h264.mp4`](https://github.com/LukasBommes/mv-extractor/blob/master/vid_h264.mp4) from the repo. #### Command Line ```bash # Extract motion vectors and show live preview extract_mvs vid_h264.mp4 --preview --verbose # Extract motion vectors and skip frame decoding (faster) extract_mvs vid_h264.mp4 --verbose --skip-decoding-frames # Extract and store motion vectors and frames to disk without showing live preview extract_mvs vid_h264.mp4 --dump # See all available options extract_mvs -h ``` #### Python API ```python from mvextractor.videocap import VideoCap cap = VideoCap() cap.open("vid_h264.mp4") # (optional) skip decoding frames cap.set_decode_frames(False) while True: ret, frame, motion_vectors, frame_type = cap.read() if not ret: break print(f"Num. motion vectors: {len(motion_vectors)}") print(f"Frame type: {frame_type}") if frame is not None: print(f"Frame size: {frame.shape}") cap.release() ``` ## Advanced Usage ### Installation via Docker Instead of installing the motion vector extractor via PyPI you can also use the prebuild Docker image from [DockerHub](https://hub.docker.com/r/lubo1994/mv-extractor). The Docker image contains the motion vector extractor and all its dependencies and comes in handy for quick testing or in case your platform is not compatible with the provided Python package. #### Prerequisites To use the Docker image you need to install [Docker](https://docs.docker.com/). Furthermore, you need to clone the source code with ```bash git clone https://github.com/LukasBommes/mv-extractor.git mv_extractor ``` #### Run Motion Vector Extraction in Docker Afterwards, you can run the extraction script in the `mv_extractor` directory as follows ```bash ./run.sh python3.12 extract_mvs.py vid_h264.mp4 --preview --verbose ``` This pulls the prebuild Docker image from DockerHub and runs the extraction script inside the Docker container. #### Building the Docker Image Locally (Optional) This step is not required and for faster installation, we recommend using the prebuilt image. If you still want to build the Docker image locally, you can do so by running the following command in the `mv_extractor` directory ```bash docker build . --tag=mv-extractor ``` Note that building can take more than one hour. Now, run the docker container with ```bash docker run -it --ipc=host --env="DISPLAY" -v $(pwd):/home/video_cap -v /tmp/.X11-unix:/tmp/.X11-unix:rw mv-extractor /bin/bash ``` ## Python API This module provides a Python API which is very similar to that of OpenCV [VideoCapture](https://docs.opencv.org/4.1.0/d8/dfe/classcv_1_1VideoCapture.html). Using the Python API is the recommended way of using the H.264 Motion Vector Capture class. #### Class :: VideoCap() | Methods | Description | | --- | --- | | VideoCap() | Constructor | | open() | Open a video file or url | | grab() | Reads the next video frame and motion vectors from the stream | | retrieve() | Decodes and returns the grabbed frame and motion vectors | | read() | Convenience function which combines a call of grab() and retrieve() | | release() | Close a video file or url and release all ressources | | set_decode_frames() | Enable/disable decoding of video frames | | Attributes | Description | | --- | --- | | decode_frames | Getter to check if frame decoding is enabled (True) or skipped (False) | ##### Method :: VideoCap() Constructor. Takes no input arguments and returns nothing. ##### Method :: open() Open a video file or url. The stream must be H264 encoded. Otherwise, undesired behaviour is likely. | Parameter | Type | Description | | --- | --- | --- | | url | string | Relative or fully specified file path or an url specifying the location of the video stream. Example "vid.flv" for a video file located in the same directory as the source files. Or "rtsp://xxx.xxx.xxx.xxx:554" for an IP camera streaming via RTSP. | | Returns | Type | Description | | --- | --- | --- | | success | bool | True if video file or url could be opened successfully, false otherwise. | ##### Method :: grab() Reads the next video frame and motion vectors from the stream, but does not yet decode it. Thus, grab() is fast. A subsequent call to retrieve() is needed to decode and return the frame and motion vectors. the purpose of splitting up grab() and retrieve() is to provide a means to capture frames in multi-camera scenarios which are as close in time as possible. To do so, first call grab() on all cameras and afterwards call retrieve() on all cameras. Takes no input arguments. | Returns | Type | Description | | --- | --- | --- | | success | bool | True if next frame and motion vectors could be grabbed successfully, false otherwise. | ##### Method :: retrieve() Decodes and returns the grabbed frame and motion vectors. Prior to calling retrieve() on a stream, grab() needs to have been called and returned successfully. Takes no input arguments and returns a tuple with the elements described in the table below. | Index | Name | Type | Description | | --- | --- | --- | --- | | 0 | success | bool | True in case the frame and motion vectors could be retrieved sucessfully, false otherwise or in case the end of stream is reached. When false, the other tuple elements are set to empty numpy arrays or 0. | | 1 | frame | numpy array | Array of dtype uint8 shape (h, w, 3) containing the decoded video frame. w and h are the width and height of this frame in pixels. Channels are in BGR order. If no frame could be decoded an empty numpy ndarray of shape (0, 0, 3) and dtype uint8 is returned. If frame decoding is disabled with set_decode_frames(False) None is returned instead. | | 2 | motion vectors | numpy array | Array of dtype int32 and shape (N, 10) containing the N motion vectors of the frame. Each row of the array corresponds to one motion vector. If no motion vectors are present in a frame, e.g. if the frame is an `I` frame an empty numpy array of shape (0, 10) and dtype int32 is returned. The columns of each vector have the following meaning (also refer to [AVMotionVector](https://ffmpeg.org/doxygen/4.1/structAVMotionVector.html) in FFMPEG documentation):
- 0: `source`: offset of the reference frame from the current frame. The reference frame is the frame where the motion vector points to and where the corresponding macroblock comes from. If `source < 0`, the reference frame is in the past. For `source > 0` the it is in the future (in display order).
- 1: `w`: width of the vector's macroblock.
- 2: `h`: height of the vector's macroblock.
- 3: `src_x`: x-location (in pixels) where the motion vector points to in the reference frame.
- 4: `src_y`: y-location (in pixels) where the motion vector points to in the reference frame.
- 5: `dst_x`: x-location of the vector's origin in the current frame (in pixels). Corresponds to the x-center coordinate of the corresponding macroblock.
- 6: `dst_y`: y-location of the vector's origin in the current frame (in pixels). Corresponds to the y-center coordinate of the corresponding macroblock.
- 7: `motion_x`: Macroblock displacement in x-direction, multiplied by `motion_scale` to become integer. Used to compute fractional value for `src_x` as `src_x = dst_x + motion_x / motion_scale`.
- 8: `motion_y`: Macroblock displacement in y-direction, multiplied by `motion_scale` to become integer. Used to compute fractional value for `src_y` as `src_y = dst_y + motion_y / motion_scale`.
- 9: `motion_scale`: see definiton of columns 7 and 8. Used to scale up the motion components to integer values. E.g. if `motion_scale = 4`, motion components can be integer values but encode a float with 1/4 pixel precision.

Note: `src_x` and `src_y` are only in integer resolution. They are contained in the [AVMotionVector](https://ffmpeg.org/doxygen/4.1/structAVMotionVector.html) struct and exported only for the sake of completeness. Use equations in field 7 and 8 to get more accurate fractional values for `src_x` and `src_y`. | | 3 | frame_type | string | Unicode string representing the type of frame. Can be `"I"` for a keyframe, `"P"` for a frame with references to only past frames and `"B"` for a frame with references to both past and future frames. A `"?"` string indicates an unknown frame type. | ##### Method :: read() Convenience function which internally calls first grab() and then retrieve(). It takes no arguments and returns the same values as retrieve(). ##### Method :: release() Close a video file or url and release all ressources. Takes no input arguments and returns nothing. ##### Method :: set_decode_frames() Enable/disable decoding of video frames. May be called anytime, even mid-stream. Returns nothing. | Parameter | Type | Description | | --- | --- | --- | | enable | bool | If True (default) RGB frames are decoded and returned in addition to extracted motion vectors. If False, frame decoding is skipped, yielding much higher extraction througput. | ## C++ API The C++ API differs from the Python API in what parameters the methods expect and what values they return. Refer to the docstrings in `src/video_cap.hpp`. ## Theory What follows is a short explanation of the data returned by the `VideoCap` class. Also refer this [excellent book](https://dl.acm.org/citation.cfm?id=1942939) by Iain E. Richardson for more details. ##### Frame The decoded video frame. Nothing special about that. ##### Motion Vectors H.264 and MPEG-4 Part 2 use different techniques to reduce the size of a raw video frame prior to sending it over a network or storing it into a file. One of those techniques is motion estimation and prediction of future frames based on previous or future frames. Each frame is segmented into macroblocks of e.g. 16 pixel x 16 pixel. During encoding motion estimation matches every macroblock to a similar looking macroblock in a previously encoded frame (note that this frame can also be a future frame since encoding and presentation order might differ). This allows to transmit only those motion vectors and the reference macroblock instead of all macroblocks, effectively reducing the amount of transmitted or stored data.
Motion vectors correlate directly with motion in the video scene and are useful for various computer vision tasks, such as visual object tracking. In MPEG-4 Part 2 macroblocks are always 16 pixel x 16 pixel. In H.264 macroblocks can be 16x16, 16x8, 8x16, 8x8, 8x4, 4x8, or 4x4 in size. ##### Frame Types The frame type is either "P", "B" or "I" and refers to the H.264 encoding mode of the current frame. An "I" frame is send fully over the network and serves as a reference for "P" and "B" frames for which only differences to previously decoded frames are transmitted. Those differences are encoded via motion vectors. As a consequence, for an "I" frame no motion vectors are returned by this library. The difference between "P" and "B" frames is that "P" frames refer only to past frames, whereas "B" frames have motion vectors which refer to both past and future frames. References to future frames are possible even with live streams because the decoding order of frames differs from the presentation order. ## About This software is maintained by [**Lukas Bommes**](https://lukasbommes.de/). It is based on [MV-Tractus](https://github.com/jishnujayakumar/MV-Tractus/tree/master/include) and OpenCV's [videoio module](https://github.com/opencv/opencv/tree/master/modules/videoio). #### License This project is licensed under the MIT License - see the [LICENSE](https://github.com/LukasBommes/mv-extractor/blob/master/LICENSE) file for details. #### Citation If you use our work for academic research please cite ``` @INPROCEEDINGS{9248145, author={L. {Bommes} and X. {Lin} and J. {Zhou}}, booktitle={2020 15th IEEE Conference on Industrial Electronics and Applications (ICIEA)}, title={MVmed: Fast Multi-Object Tracking in the Compressed Domain}, year={2020}, volume={}, number={}, pages={1419-1424}, doi={10.1109/ICIEA48937.2020.9248145}} ``` ================================================ FILE: dockerhub.md ================================================ # Motion Vector Extractor The [motion vector extractor](https://github.com/LukasBommes/mv-extractor) is a tool to extract frames, motion vectors and frame types from H.264 and MPEG-4 Part 2 encoded videos. The tool provides a single class, which serves as a replacement for OpenCV's [VideoCapture](https://docs.opencv.org/4.1.0/d8/dfe/classcv_1_1VideoCapture.html) and can be used to read and decode video frames from a H.264 or MPEG-4 Part 2 encoded video stream/file. This Docker image is based on the [manylinux_2_28](https://github.com/pypa/manylinux) image and serves two purposes: 1. It contains all dependencies to run the motion vector extractor and its test suite. 2. It functions as build environment for building the [Python package](https://pypi.org/project/motion-vector-extractor/) of the motion vector extraction for all supported Python versions. ## Tags with respective Dockerfile links - [`v1.1.0`, `latest`](https://github.com/LukasBommes/mv-extractor/blob/c56b94b9ec7e96e273e67eb5cf19f0e6b927f68b/Dockerfile) - [`v1.0.6`](https://github.com/LukasBommes/mv-extractor/blob/75424afe230f9847f3e86e243f46d3105eeba858/Dockerfile) - [`v1.0.5`](https://github.com/LukasBommes/mv-extractor/blob/ac539243f6cd7cc1d9640d8ce52ba1814a3cbc7d/Dockerfile) - [`v1.0.4`](https://github.com/LukasBommes/mv-extractor/blob/94a79e0ce72446beb7b3862f8ed04a1cbce0d1a3/Dockerfile) - [`v1.0.3`](https://github.com/LukasBommes/mv-extractor/blob/2ccce5b85e1c9cf813271e443490981c5773dc02/Dockerfile) - [`v1.0.2`](https://github.com/LukasBommes/mv-extractor/blob/4dc77fe5681d55820b43657c63c81294bf47a0bc/Dockerfile) - [`v1.0.1`](https://github.com/LukasBommes/mv-extractor/blob/17ae26680194b49996e01397871bef857064514f/Dockerfile) - [`v1.0.0`](https://github.com/LukasBommes/mv-extractor/blob/4b44302a44e78618aeabde95ee02cecee311b456/Dockerfile) Images tagged with `dev` and `buildcache` are intermediate artefacts generated by the CI and should not be used directly. ## Usage Pull and run the motion vector extractor with ```cmd docker run lubo1994/mv-extractor:latest extract_mvs -h ``` Map a video file into the container and extract motion vectors (replace with the actual filename) ```cmd docker run -v ./:/home/video_cap/ lubo1994/mv-extractor:latest extract_mvs --verbose ``` If you want to use the graphical preview, you have to supply additional arguments to the docker run command ```cmd docker run -it --ipc=host --env="DISPLAY" -v ./:/home/video_cap/ -v /tmp/.X11-unix:/tmp/.X11-unix:rw lubo1994/mv-extractor:latest extract_mvs --preview ``` For more details on the usage see the [project homepage](https://github.com/LukasBommes/mv-extractor). ## About This software is written by [**Lukas Bommes**](https://lukasbommes.de/) and licensed under the [MIT License](https://github.com/LukasBommes/mv-extractor/blob/master/LICENSE). If you use the project for academic research please cite ```text @INPROCEEDINGS{9248145, author={L. {Bommes} and X. {Lin} and J. {Zhou}}, booktitle={2020 15th IEEE Conference on Industrial Electronics and Applications (ICIEA)}, title={MVmed: Fast Multi-Object Tracking in the Compressed Domain}, year={2020}, volume={}, number={}, pages={1419-1424}, doi={10.1109/ICIEA48937.2020.9248145}} ``` ================================================ FILE: extract_mvs.py ================================================ from mvextractor.__main__ import main if __name__ == "__main__": main() ================================================ FILE: pyproject.toml ================================================ [build-system] requires = [ "setuptools>=61.0", "pkgconfig>=1.5.1", "numpy>=2.0.0" ] build-backend = "setuptools.build_meta" ================================================ FILE: release.md ================================================ # Create a new release ### Step 1) Bump version Bump the version in `setup.py` ### Step 2) Push code Make changes, commit and push. ### Step 3) Run build workflow On GitHub go to the repo's "Actions" tab and manually trigger the "build" workflow. The build workflow builds the Docker image and wheels. The Docker image is automatically pushed to Dockerhub. The wheels need to be manually uploaded to PyPI as explained below. ### Step 4) Create tag and release Now, create a tag with the same version just entered in the `setup.py` and push that tag to the remote. ``` git tag vx.x.x git push origin vx.x.x ``` Then create a release on GitHub using this tag. ### Step 5) Upload wheels to PyPI First, make sure you have the most recent version of twine installed on the host ``` python3 -m pip install --upgrade twine ``` Then, download and extract the wheels from the (successfully completed) workflow run. Place them inside the "dist" folder (create if it does not exist). Then, upload to PyPI with ``` python3 -m twine upload dist/* ``` #### Step 6) Tag Docker image with correct version When pushing changes, a Docker image `lubo1994/mv-extractor:dev` is being build and pushed to DockerHub. Upon a release, this image should be tagged with the correct release version and the `latest` tag. To this end, first pull the `dev` image ``` docker pull lubo1994/mv-extractor:dev ``` and then login to the docker registry ``` cat docker_registry_password.txt | docker login --username --password-stdin ``` and tag and push the image as follows ``` docker tag lubo1994/mv-extractor:dev lubo1994/mv-extractor:vx.x.x docker push lubo1994/mv-extractor:vx.x.x docker tag lubo1994/mv-extractor:vx.x.x lubo1994/mv-extractor:latest docker push lubo1994/mv-extractor:latest ``` where `vx.x.x` is replaced with the version of the release. ================================================ FILE: run.sh ================================================ #!/bin/bash xhost + docker run \ -it \ --ipc=host \ --env="DISPLAY" \ -v $(pwd):/home/video_cap \ -v /tmp/.X11-unix:/tmp/.X11-unix:rw \ lubo1994/mv-extractor:latest \ "$@" ================================================ FILE: setup.py ================================================ from setuptools import find_packages, setup, Extension import pkgconfig from pathlib import Path import numpy as np pkgconfig_result = pkgconfig.parse('libavformat libswscale opencv4') print("Numpy dir: ", np.get_include()) mvextractor = Extension('mvextractor.videocap', include_dirs = [ *pkgconfig_result['include_dirs'], np.get_include() ], library_dirs = pkgconfig_result['library_dirs'], libraries = pkgconfig_result['libraries'], sources = [ 'src/mvextractor/py_video_cap.cpp', 'src/mvextractor/video_cap.cpp', 'src/mvextractor/mat_to_ndarray.cpp' ], extra_compile_args = ['-std=c++11'], extra_link_args = ['-fPIC', '-Wl,-Bsymbolic']) setup( name='motion-vector-extractor', author='Lukas Bommes', author_email=' ', version="2.0.0", license='MIT', url='https://github.com/LukasBommes/mv-extractor', description=('Reads video frames and MPEG-4/H.264 motion vectors.'), long_description=(Path(__file__).parent / "README.md").read_text(), long_description_content_type='text/markdown', classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", "Environment :: X11 Applications", "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Intended Audience :: Education", "Intended Audience :: Information Technology", "Topic :: Multimedia :: Video", "Topic :: Multimedia :: Video :: Capture", "Topic :: Multimedia :: Video :: Display", "License :: OSI Approved :: MIT License", "Natural Language :: English", "Operating System :: POSIX :: Linux", "Programming Language :: C", "Programming Language :: C++", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", ], keywords=['motion vector', 'video capture', 'mpeg4', 'h.264', 'compressed domain'], ext_modules=[mvextractor], packages=find_packages(where='src'), package_dir={'': 'src'}, entry_points={ 'console_scripts': [ 'extract_mvs=mvextractor.__main__:main', ], }, python_requires='>=3.9, <4', # minimum versions of numpy and opencv are the oldest versions # just supporting the minimum Python version (Python 3.9) install_requires=['numpy>=1.19.3', 'opencv-python>=4.4.0.46'] ) ================================================ FILE: src/mvextractor/__init__.py ================================================ ================================================ FILE: src/mvextractor/__main__.py ================================================ import sys import os import time from datetime import datetime import argparse import numpy as np import cv2 from mvextractor.videocap import VideoCap def draw_motion_vectors(frame, motion_vectors): if len(motion_vectors) > 0: num_mvs = np.shape(motion_vectors)[0] shift = 2 factor = (1 << shift) for mv in np.split(motion_vectors, num_mvs): start_pt = (int((mv[0, 5] + mv[0, 7] / mv[0, 9]) * factor + 0.5), int((mv[0, 6] + mv[0, 8] / mv[0, 9]) * factor + 0.5)) end_pt = (mv[0, 5] * factor, mv[0, 6] * factor) cv2.arrowedLine(frame, start_pt, end_pt, (0, 0, 255), 1, cv2.LINE_AA, shift, 0.1) return frame def main(args=None): if args is None: args = sys.argv[1:] parser = argparse.ArgumentParser(description='Extract motion vectors from video.') parser.add_argument('video_url', type=str, nargs='?', help='file path or url of the video stream') parser.add_argument('-p', '--preview', action='store_true', help='show a preview video with overlaid motion vectors') parser.add_argument('-v', '--verbose', action='store_true', help='show detailled text output') parser.add_argument('-s', '--skip-decoding-frames', action='store_true', help='skip decoding RGB frames and return only motion vectors (faster)') parser.add_argument('-d', '--dump', nargs='?', const=True, help='dump frames, motion vectors and frame types to optionally specified output directory') args = parser.parse_args() if args.dump: if isinstance(args.dump, str): dumpdir = args.dump else: dumpdir = f"out-{datetime.now().strftime('%Y-%m-%dT%H:%M:%S')}" for child in ["frames", "motion_vectors"]: os.makedirs(os.path.join(dumpdir, child), exist_ok=True) cap = VideoCap() # open the video file ret = cap.open(args.video_url) if not ret: raise RuntimeError(f"Could not open {args.video_url}") if args.verbose: print("Sucessfully opened video file") if args.skip_decoding_frames: cap.set_decode_frames(False) step = 0 times = [] # continuously read and display video frames and motion vectors while True: if args.verbose: print("Frame: ", step, end=" ") tstart = time.perf_counter() # read next video frame and corresponding motion vectors ret, frame, motion_vectors, frame_type = cap.read() tend = time.perf_counter() telapsed = tend - tstart times.append(telapsed) # if there is an error reading the frame if not ret: if args.verbose: print("No frame read. Stopping.") break # print results if args.verbose: print("frame type: {} | ".format(frame_type), end=" ") if frame is not None: print("frame size: {} | ".format(np.shape(frame)), end=" ") else: print("frame size: () | ", end=" ") print("motion vectors: {} | ".format(np.shape(motion_vectors)), end=" ") print("elapsed time: {} s".format(telapsed)) # draw vectors on frames if not args.skip_decoding_frames and frame is not None: frame = draw_motion_vectors(frame, motion_vectors) # store motion vectors, frames, and fraem types in output directory if args.dump: np.save(os.path.join(dumpdir, "motion_vectors", f"mvs-{step}.npy"), motion_vectors) with open(os.path.join(dumpdir, "frame_types.txt"), "a") as f: f.write(frame_type+"\n") if not args.skip_decoding_frames and frame is not None: cv2.imwrite(os.path.join(dumpdir, "frames", f"frame-{step}.jpg"), frame) step += 1 if args.preview and not args.skip_decoding_frames: cv2.imshow("Frame", frame) # if user presses "q" key stop program if cv2.waitKey(1) & 0xFF == ord('q'): break if args.verbose: print("average dt: ", np.mean(times)) cap.release() # close the GUI window if args.preview: cv2.destroyAllWindows() if __name__ == "__main__": sys.exit(main()) ================================================ FILE: src/mvextractor/mat_to_ndarray.cpp ================================================ // Taken from OpenCV master commit e2a5a6a05c7ce64911e1e898e986abe8dd26cab6 // File: opencv/modules/python/cv2.cpp #include "mat_to_ndarray.hpp" class PyAllowThreads { public: PyAllowThreads() : _state(PyEval_SaveThread()) {} ~PyAllowThreads() { PyEval_RestoreThread(_state); } private: PyThreadState* _state; }; class PyEnsureGIL { public: PyEnsureGIL() : _state(PyGILState_Ensure()) {} ~PyEnsureGIL() { PyGILState_Release(_state); } private: PyGILState_STATE _state; }; #define ERRWRAP2(expr) \ try \ { \ PyAllowThreads allowThreads; \ expr; \ } \ catch (const cv::Exception &e) \ { \ PyObject_SetAttrString(opencv_error, "file", PyString_FromString(e.file.c_str())); \ PyObject_SetAttrString(opencv_error, "func", PyString_FromString(e.func.c_str())); \ PyObject_SetAttrString(opencv_error, "line", PyInt_FromLong(e.line)); \ PyObject_SetAttrString(opencv_error, "code", PyInt_FromLong(e.code)); \ PyObject_SetAttrString(opencv_error, "msg", PyString_FromString(e.msg.c_str())); \ PyObject_SetAttrString(opencv_error, "err", PyString_FromString(e.err.c_str())); \ PyErr_SetString(opencv_error, e.what()); \ return 0; \ } using namespace cv; class NumpyAllocator : public MatAllocator { public: NumpyAllocator() { stdAllocator = Mat::getStdAllocator(); } ~NumpyAllocator() {} UMatData* allocate(PyObject* o, int dims, const int* sizes, int type, size_t* step) const { UMatData* u = new UMatData(this); u->data = u->origdata = (uchar*)PyArray_DATA((PyArrayObject*) o); npy_intp* _strides = PyArray_STRIDES((PyArrayObject*) o); for( int i = 0; i < dims - 1; i++ ) step[i] = (size_t)_strides[i]; step[dims-1] = CV_ELEM_SIZE(type); u->size = sizes[0]*step[0]; u->userdata = o; return u; } UMatData* allocate(int dims0, const int* sizes, int type, void* data, size_t* step, AccessFlag flags, UMatUsageFlags usageFlags) const CV_OVERRIDE { if( data != 0 ) { // issue #6969: CV_Error(Error::StsAssert, "The data should normally be NULL!"); // probably this is safe to do in such extreme case return stdAllocator->allocate(dims0, sizes, type, data, step, flags, usageFlags); } PyEnsureGIL gil; int depth = CV_MAT_DEPTH(type); int cn = CV_MAT_CN(type); const int f = (int)(sizeof(size_t)/8); int typenum = depth == CV_8U ? NPY_UBYTE : depth == CV_8S ? NPY_BYTE : depth == CV_16U ? NPY_USHORT : depth == CV_16S ? NPY_SHORT : depth == CV_32S ? NPY_INT : depth == CV_32F ? NPY_FLOAT : depth == CV_64F ? NPY_DOUBLE : f*NPY_ULONGLONG + (f^1)*NPY_UINT; int i, dims = dims0; cv::AutoBuffer _sizes(dims + 1); for( i = 0; i < dims; i++ ) _sizes[i] = sizes[i]; if( cn > 1 ) _sizes[dims++] = cn; PyObject* o = PyArray_SimpleNew(dims, _sizes.data(), typenum); if(!o) CV_Error_(Error::StsError, ("The numpy array of typenum=%d, ndims=%d can not be created", typenum, dims)); return allocate(o, dims0, sizes, type, step); } bool allocate(UMatData* u, AccessFlag accessFlags, UMatUsageFlags usageFlags) const CV_OVERRIDE { return stdAllocator->allocate(u, accessFlags, usageFlags); } void deallocate(UMatData* u) const CV_OVERRIDE { if(!u) return; PyEnsureGIL gil; CV_Assert(u->urefcount >= 0); CV_Assert(u->refcount >= 0); if(u->refcount == 0) { PyObject* o = (PyObject*)u->userdata; Py_XDECREF(o); delete u; } } const MatAllocator* stdAllocator; }; NumpyAllocator g_numpyAllocator; int* NDArrayConverter::init() { import_array(); return NULL; } NDArrayConverter::NDArrayConverter() { init(); } PyObject* NDArrayConverter::toNDArray(const cv::Mat& m) { if( !m.data ) Py_RETURN_NONE; Mat temp, *p = (Mat*)&m; if(!p->u || p->allocator != &g_numpyAllocator) { temp.allocator = &g_numpyAllocator; ERRWRAP2(m.copyTo(temp)); p = &temp; } PyObject* o = (PyObject*)p->u->userdata; Py_INCREF(o); return o; } ================================================ FILE: src/mvextractor/mat_to_ndarray.hpp ================================================ // Taken from OpenCV master commit e2a5a6a05c7ce64911e1e898e986abe8dd26cab6 // File: opencv/modules/python/cv2.cpp #include #include #include #include "opencv2/core/core.hpp" #include "opencv2/core/types_c.h" #include "opencv2/opencv_modules.hpp" #include "pycompat.hpp" #include static PyObject* opencv_error = NULL; class PyAllowThreads; class PyEnsureGIL; #define ERRWRAP2(expr) \ try \ { \ PyAllowThreads allowThreads; \ expr; \ } \ catch (const cv::Exception &e) \ { \ PyObject_SetAttrString(opencv_error, "file", PyString_FromString(e.file.c_str())); \ PyObject_SetAttrString(opencv_error, "func", PyString_FromString(e.func.c_str())); \ PyObject_SetAttrString(opencv_error, "line", PyInt_FromLong(e.line)); \ PyObject_SetAttrString(opencv_error, "code", PyInt_FromLong(e.code)); \ PyObject_SetAttrString(opencv_error, "msg", PyString_FromString(e.msg.c_str())); \ PyObject_SetAttrString(opencv_error, "err", PyString_FromString(e.err.c_str())); \ PyErr_SetString(opencv_error, e.what()); \ return 0; \ } class NumpyAllocator; enum { ARG_NONE = 0, ARG_MAT = 1, ARG_SCALAR = 2 }; class NDArrayConverter { private: int* init(); public: NDArrayConverter(); PyObject* toNDArray(const cv::Mat& m); }; ================================================ FILE: src/mvextractor/py_video_cap.cpp ================================================ #define PY_SSIZE_T_CLEAN #include #include #include #include "video_cap.hpp" #include "mat_to_ndarray.hpp" typedef struct { PyObject_HEAD VideoCap vcap; } VideoCapObject; static int VideoCap_init(VideoCapObject *self, PyObject *args, PyObject *kwds) { new(&self->vcap) VideoCap(); return 0; } static void VideoCap_dealloc(VideoCapObject *self) { self->vcap.release(); Py_TYPE(self)->tp_free((PyObject *) self); } static PyObject * VideoCap_open(VideoCapObject *self, PyObject *args) { const char *url; if (!PyArg_ParseTuple(args, "s", &url)) Py_RETURN_FALSE; if (!self->vcap.open(url)) Py_RETURN_FALSE; Py_RETURN_TRUE; } static PyObject * VideoCap_grab(VideoCapObject *self, PyObject *Py_UNUSED(ignored)) { if (!self->vcap.grab()) Py_RETURN_FALSE; Py_RETURN_TRUE; } static PyObject * VideoCap_retrieve(VideoCapObject *self, PyObject *Py_UNUSED(ignored)) { cv::Mat frame_cv; uint8_t *frame = NULL; int width = 0; int height = 0; int step = 0; int cn = 0; MVS_DTYPE *motion_vectors = NULL; MVS_DTYPE num_mvs = 0; char frame_type[2] = "?"; PyObject *ret = Py_True; if (!self->vcap.retrieve(&frame, &step, &width, &height, &cn, frame_type, &motion_vectors, &num_mvs)) { num_mvs = 0; width = 0; height = 0; step = 0; cn = 0; ret = Py_False; } // copy frame buffer into new cv::Mat PyObject* frame_nd = Py_None; if (self->vcap.getDecodeFrames()) { cv::Mat(height, width, CV_MAKETYPE(CV_8U, cn), frame, step).copyTo(frame_cv); // convert frame cv::Mat to numpy.ndarray NDArrayConverter cvt; frame_nd = cvt.toNDArray(frame_cv); } else { Py_INCREF(Py_None); } // convert motion vector buffer into numpy array npy_intp dims_mvs[2] = {(npy_intp)num_mvs, 10}; PyObject *motion_vectors_nd = PyArray_SimpleNewFromData(2, dims_mvs, MVS_DTYPE_NP, motion_vectors); PyArray_ENABLEFLAGS((PyArrayObject*)motion_vectors_nd, NPY_ARRAY_OWNDATA); return Py_BuildValue("(ONNs)", ret, frame_nd, motion_vectors_nd, (const char*)frame_type); } static PyObject * VideoCap_read(VideoCapObject *self, PyObject *Py_UNUSED(ignored)) { cv::Mat frame_cv; uint8_t *frame = NULL; int width = 0; int height = 0; int step = 0; int cn = 0; MVS_DTYPE *motion_vectors = NULL; MVS_DTYPE num_mvs = 0; char frame_type[2] = "?"; PyObject *ret = Py_True; if (!self->vcap.read(&frame, &step, &width, &height, &cn, frame_type, &motion_vectors, &num_mvs)) { num_mvs = 0; width = 0; height = 0; step = 0; cn = 0; ret = Py_False; } PyObject* frame_nd = Py_None; if (self->vcap.getDecodeFrames()) { cv::Mat(height, width, CV_MAKETYPE(CV_8U, cn), frame, step).copyTo(frame_cv); // convert frame cv::Mat to numpy.ndarray NDArrayConverter cvt; frame_nd = cvt.toNDArray(frame_cv); } else { Py_INCREF(Py_None); } // convert motion vector buffer into numpy array npy_intp dims_mvs[2] = {(npy_intp)num_mvs, 10}; PyObject *motion_vectors_nd = PyArray_SimpleNewFromData(2, dims_mvs, MVS_DTYPE_NP, motion_vectors); PyArray_ENABLEFLAGS((PyArrayObject*)motion_vectors_nd, NPY_ARRAY_OWNDATA); return Py_BuildValue("(ONNs)", ret, frame_nd, motion_vectors_nd, (const char*)frame_type); } static PyObject * VideoCap_release(VideoCapObject *self, PyObject *Py_UNUSED(ignored)) { self->vcap.release(); Py_RETURN_NONE; } static PyObject * VideoCap_set_decode_frames(VideoCapObject *self, PyObject *args) { int enable = 0; if (!PyArg_ParseTuple(args, "p", &enable)) Py_RETURN_NONE; self->vcap.setDecodeFrames(enable != 0); Py_RETURN_NONE; } static PyObject * VideoCap_get_decode_frames(VideoCapObject *self, PyObject *Py_UNUSED(ignored)) { if (self->vcap.getDecodeFrames()) Py_RETURN_TRUE; else Py_RETURN_FALSE; } static PyMethodDef VideoCap_methods[] = { {"open", (PyCFunction) VideoCap_open, METH_VARARGS, "Open a video file or device with given filename/url"}, {"read", (PyCFunction) VideoCap_read, METH_NOARGS, "Grab and decode the next frame and motion vectors"}, {"grab", (PyCFunction) VideoCap_grab, METH_NOARGS, "Grab the next frame and motion vectors from the stream"}, {"retrieve", (PyCFunction) VideoCap_retrieve, METH_NOARGS, "Decode the grabbed frame and motion vectors"}, {"release", (PyCFunction) VideoCap_release, METH_NOARGS, "Release the video device and free ressources"}, {"set_decode_frames", (PyCFunction) VideoCap_set_decode_frames, METH_VARARGS, "Enable/disable decoding of RGB frames"}, {NULL} /* Sentinel */ }; static PyGetSetDef VideoCap_getset[] = { {"decode_frames", (getter)VideoCap_get_decode_frames, NULL, "Whether RGB frames are decoded (True) or only motion vectors (False)", NULL}, {NULL} // Sentinel }; static PyTypeObject VideoCapType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "videocap.VideoCap", .tp_basicsize = sizeof(VideoCapObject), .tp_itemsize = 0, .tp_dealloc = (destructor) VideoCap_dealloc, .tp_vectorcall_offset = NULL, .tp_getattr = NULL, .tp_setattr = NULL, .tp_as_async = NULL, .tp_repr = NULL, .tp_as_number = NULL, .tp_as_sequence = NULL, .tp_as_mapping = NULL, .tp_hash = NULL, .tp_call = NULL, .tp_str = NULL, .tp_getattro = NULL, .tp_setattro = NULL, .tp_as_buffer = NULL, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_doc = "Video Capture Object", .tp_traverse = NULL, .tp_clear = NULL, .tp_richcompare = NULL, .tp_weaklistoffset = 0, .tp_iter = NULL, .tp_iternext = NULL, .tp_methods = VideoCap_methods, .tp_members = NULL, .tp_getset = VideoCap_getset, .tp_base = NULL, .tp_dict = NULL, .tp_descr_get = NULL, .tp_descr_set = NULL, .tp_dictoffset = 0, .tp_init = (initproc) VideoCap_init, .tp_alloc = NULL, .tp_new = PyType_GenericNew, .tp_free = NULL, .tp_is_gc = NULL, .tp_bases = NULL, .tp_mro = NULL, .tp_cache = NULL, .tp_subclasses = NULL, .tp_weaklist = NULL, .tp_del = NULL, .tp_version_tag = 0, .tp_finalize = NULL, }; static PyModuleDef videocapmodule = { PyModuleDef_HEAD_INIT, .m_name = "videocap", .m_doc = "Capture video frames and motion vectors from a H264 encoded stream.", .m_size = -1, }; PyMODINIT_FUNC PyInit_videocap(void) { Py_Initialize(); // maybe not needed import_array(); PyObject *m; if (PyType_Ready(&VideoCapType) < 0) return NULL; m = PyModule_Create(&videocapmodule); if (m == NULL) return NULL; Py_INCREF(&VideoCapType); PyModule_AddObject(m, "VideoCap", (PyObject *) &VideoCapType); return m; } ================================================ FILE: src/mvextractor/pycompat.hpp ================================================ // Taken from OpenCV master commit e2a5a6a05c7ce64911e1e898e986abe8dd26cab6 // File: opencv/modules/python/pycompat.hpp.cpp /*M/////////////////////////////////////////////////////////////////////////////////////// // // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. // // By downloading, copying, installing or using the software you agree to this license. // If you do not agree to this license, do not download, install, // copy or use the software. // // // License Agreement // For Open Source Computer Vision Library // // Copyright (C) 2000-2008, Intel Corporation, all rights reserved. // Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. // Third party copyrights are property of their respective owners. // // Redistribution and use in source and binary forms, with or without modification, // are permitted provided that the following conditions are met: // // * Redistribution's of source code must retain the above copyright notice, // this list of conditions and the following disclaimer. // // * Redistribution's in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimer in the documentation // and/or other materials provided with the distribution. // // * The name of the copyright holders may not be used to endorse or promote products // derived from this software without specific prior written permission. // // This software is provided by the copyright holders and contributors "as is" and // any express or implied warranties, including, but not limited to, the implied // warranties of merchantability and fitness for a particular purpose are disclaimed. // In no event shall the Intel Corporation or contributors be liable for any direct, // indirect, incidental, special, exemplary, or consequential damages // (including, but not limited to, procurement of substitute goods or services; // loss of use, data, or profits; or business interruption) however caused // and on any theory of liability, whether in contract, strict liability, // or tort (including negligence or otherwise) arising in any way out of // the use of this software, even if advised of the possibility of such damage. // //M*/ // Defines for Python 2/3 compatibility. #ifndef __PYCOMPAT_HPP__ #define __PYCOMPAT_HPP__ #if PY_MAJOR_VERSION >= 3 // Python3 treats all ints as longs, PyInt_X functions have been removed. #define PyInt_Check PyLong_Check #define PyInt_CheckExact PyLong_CheckExact #define PyInt_AsLong PyLong_AsLong #define PyInt_AS_LONG PyLong_AS_LONG #define PyInt_FromLong PyLong_FromLong #define PyNumber_Int PyNumber_Long #define PyString_FromString PyUnicode_FromString #define PyString_FromStringAndSize PyUnicode_FromStringAndSize #endif // PY_MAJOR >=3 static inline bool getUnicodeString(PyObject * obj, std::string &str) { bool res = false; if (PyUnicode_Check(obj)) { PyObject * bytes = PyUnicode_AsUTF8String(obj); if (PyBytes_Check(bytes)) { const char * raw = PyBytes_AsString(bytes); if (raw) { str = std::string(raw); res = true; } } Py_XDECREF(bytes); } #if PY_MAJOR_VERSION < 3 else if (PyString_Check(obj)) { const char * raw = PyString_AsString(obj); if (raw) { str = std::string(raw); res = true; } } #endif return res; } //================================================================================================== #define CV_PY_FN_WITH_KW_(fn, flags) (PyCFunction)(void*)(PyCFunctionWithKeywords)(fn), (flags) | METH_VARARGS | METH_KEYWORDS #define CV_PY_FN_NOARGS_(fn, flags) (PyCFunction)(fn), (flags) | METH_NOARGS #define CV_PY_FN_WITH_KW(fn) CV_PY_FN_WITH_KW_(fn, 0) #define CV_PY_FN_NOARGS(fn) CV_PY_FN_NOARGS_(fn, 0) #define CV_PY_TO_CLASS(TYPE) \ template<> \ bool pyopencv_to(PyObject* dst, TYPE& src, const char* name) \ { \ if (!dst || dst == Py_None) \ return true; \ Ptr ptr; \ \ if (!pyopencv_to(dst, ptr, name)) return false; \ src = *ptr; \ return true; \ } #define CV_PY_FROM_CLASS(TYPE) \ template<> \ PyObject* pyopencv_from(const TYPE& src) \ { \ Ptr ptr(new TYPE()); \ \ *ptr = src; \ return pyopencv_from(ptr); \ } #define CV_PY_TO_CLASS_PTR(TYPE) \ template<> \ bool pyopencv_to(PyObject* dst, TYPE*& src, const char* name) \ { \ if (!dst || dst == Py_None) \ return true; \ Ptr ptr; \ \ if (!pyopencv_to(dst, ptr, name)) return false; \ src = ptr; \ return true; \ } #define CV_PY_FROM_CLASS_PTR(TYPE) \ static PyObject* pyopencv_from(TYPE*& src) \ { \ return pyopencv_from(Ptr(src)); \ } #define CV_PY_TO_ENUM(TYPE) \ template<> \ bool pyopencv_to(PyObject* dst, TYPE& src, const char* name) \ { \ if (!dst || dst == Py_None) \ return true; \ int underlying = 0; \ \ if (!pyopencv_to(dst, underlying, name)) return false; \ src = static_cast(underlying); \ return true; \ } #define CV_PY_FROM_ENUM(TYPE) \ template<> \ PyObject* pyopencv_from(const TYPE& src) \ { \ return pyopencv_from(static_cast(src)); \ } //================================================================================================== #if PY_MAJOR_VERSION >= 3 #define CVPY_TYPE_HEAD PyVarObject_HEAD_INIT(&PyType_Type, 0) #define CVPY_TYPE_INCREF(T) Py_INCREF(T) #else #define CVPY_TYPE_HEAD PyObject_HEAD_INIT(&PyType_Type) 0, #define CVPY_TYPE_INCREF(T) _Py_INC_REFTOTAL _Py_REF_DEBUG_COMMA (T)->ob_refcnt++ #endif #define CVPY_TYPE_DECLARE(NAME, STORAGE, SNAME) \ struct pyopencv_##NAME##_t \ { \ PyObject_HEAD \ STORAGE v; \ }; \ static PyTypeObject pyopencv_##NAME##_TypeXXX = \ { \ CVPY_TYPE_HEAD \ MODULESTR"."#NAME, \ sizeof(pyopencv_##NAME##_t), \ }; \ static PyTypeObject * pyopencv_##NAME##_TypePtr = &pyopencv_##NAME##_TypeXXX; \ static bool pyopencv_##NAME##_getp(PyObject * self, STORAGE * & dst) \ { \ if (PyObject_TypeCheck(self, pyopencv_##NAME##_TypePtr)) \ { \ dst = &(((pyopencv_##NAME##_t*)self)->v); \ return true; \ } \ return false; \ } \ static PyObject * pyopencv_##NAME##_Instance(const STORAGE &r) \ { \ pyopencv_##NAME##_t *m = PyObject_NEW(pyopencv_##NAME##_t, pyopencv_##NAME##_TypePtr); \ new (&(m->v)) STORAGE(r); \ return (PyObject*)m; \ } \ static void pyopencv_##NAME##_dealloc(PyObject* self) \ { \ ((pyopencv_##NAME##_t*)self)->v.STORAGE::~SNAME(); \ PyObject_Del(self); \ } \ static PyObject* pyopencv_##NAME##_repr(PyObject* self) \ { \ char str[1000]; \ sprintf(str, "<"#NAME" %p>", self); \ return PyString_FromString(str); \ } #define CVPY_TYPE_INIT_STATIC(NAME, ERROR_HANDLER, BASE, CONSTRUCTOR) \ { \ pyopencv_##NAME##_TypePtr->tp_base = pyopencv_##BASE##_TypePtr; \ pyopencv_##NAME##_TypePtr->tp_dealloc = pyopencv_##NAME##_dealloc; \ pyopencv_##NAME##_TypePtr->tp_repr = pyopencv_##NAME##_repr; \ pyopencv_##NAME##_TypePtr->tp_getset = pyopencv_##NAME##_getseters; \ pyopencv_##NAME##_TypePtr->tp_init = (initproc) CONSTRUCTOR; \ pyopencv_##NAME##_TypePtr->tp_methods = pyopencv_##NAME##_methods; \ pyopencv_##NAME##_TypePtr->tp_alloc = PyType_GenericAlloc; \ pyopencv_##NAME##_TypePtr->tp_new = PyType_GenericNew; \ pyopencv_##NAME##_TypePtr->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; \ if (PyType_Ready(pyopencv_##NAME##_TypePtr) != 0) \ { \ ERROR_HANDLER; \ } \ CVPY_TYPE_INCREF(pyopencv_##NAME##_TypePtr); \ PyModule_AddObject(m, #NAME, (PyObject *)pyopencv_##NAME##_TypePtr); \ } //================================================================================================== #define CVPY_TYPE_DECLARE_DYNAMIC(NAME, STORAGE, SNAME) \ struct pyopencv_##NAME##_t \ { \ PyObject_HEAD \ STORAGE v; \ }; \ static PyObject * pyopencv_##NAME##_TypePtr = 0; \ static bool pyopencv_##NAME##_getp(PyObject * self, STORAGE * & dst) \ { \ if (PyObject_TypeCheck(self, (PyTypeObject*)pyopencv_##NAME##_TypePtr)) \ { \ dst = &(((pyopencv_##NAME##_t*)self)->v); \ return true; \ } \ return false; \ } \ static PyObject * pyopencv_##NAME##_Instance(const STORAGE &r) \ { \ pyopencv_##NAME##_t *m = PyObject_New(pyopencv_##NAME##_t, (PyTypeObject*)pyopencv_##NAME##_TypePtr); \ new (&(m->v)) STORAGE(r); \ return (PyObject*)m; \ } \ static void pyopencv_##NAME##_dealloc(PyObject* self) \ { \ ((pyopencv_##NAME##_t*)self)->v.STORAGE::~SNAME(); \ PyObject_Del(self); \ } \ static PyObject* pyopencv_##NAME##_repr(PyObject* self) \ { \ char str[1000]; \ sprintf(str, "<"#NAME" %p>", self); \ return PyString_FromString(str); \ } \ static PyType_Slot pyopencv_##NAME##_Slots[] = \ { \ {Py_tp_dealloc, 0}, \ {Py_tp_repr, 0}, \ {Py_tp_getset, 0}, \ {Py_tp_init, 0}, \ {Py_tp_methods, 0}, \ {Py_tp_alloc, 0}, \ {Py_tp_new, 0}, \ {0, 0} \ }; \ static PyType_Spec pyopencv_##NAME##_Spec = \ { \ MODULESTR"."#NAME, \ sizeof(pyopencv_##NAME##_t), \ 0, \ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, \ pyopencv_##NAME##_Slots \ }; #define CVPY_TYPE_INIT_DYNAMIC(NAME, ERROR_HANDLER, BASE, CONSTRUCTOR) \ { \ pyopencv_##NAME##_Slots[0].pfunc /*tp_dealloc*/ = (void*)pyopencv_##NAME##_dealloc; \ pyopencv_##NAME##_Slots[1].pfunc /*tp_repr*/ = (void*)pyopencv_##NAME##_repr; \ pyopencv_##NAME##_Slots[2].pfunc /*tp_getset*/ = (void*)pyopencv_##NAME##_getseters; \ pyopencv_##NAME##_Slots[3].pfunc /*tp_init*/ = (void*) CONSTRUCTOR; \ pyopencv_##NAME##_Slots[4].pfunc /*tp_methods*/ = pyopencv_##NAME##_methods; \ pyopencv_##NAME##_Slots[5].pfunc /*tp_alloc*/ = (void*)PyType_GenericAlloc; \ pyopencv_##NAME##_Slots[6].pfunc /*tp_new*/ = (void*)PyType_GenericNew; \ PyObject * bases = 0; \ if (pyopencv_##BASE##_TypePtr) \ bases = PyTuple_Pack(1, pyopencv_##BASE##_TypePtr); \ pyopencv_##NAME##_TypePtr = PyType_FromSpecWithBases(&pyopencv_##NAME##_Spec, bases); \ if (!pyopencv_##NAME##_TypePtr) \ { \ printf("Failed to init: " #NAME ", base (" #BASE ")" "\n"); \ ERROR_HANDLER; \ } \ PyModule_AddObject(m, #NAME, (PyObject *)pyopencv_##NAME##_TypePtr); \ } // Debug module load: // // else \ // { \ // printf("Init: " #NAME ", base (" #BASE ") -> %p" "\n", pyopencv_##NAME##_TypePtr); \ // } \ #endif // END HEADER GUARD ================================================ FILE: src/mvextractor/video_cap.cpp ================================================ #include "video_cap.hpp" VideoCap::VideoCap() { this->opts = NULL; this->codec = NULL; this->fmt_ctx = NULL; this->video_dec_ctx = NULL; this->video_stream = NULL; this->video_stream_idx = -1; this->frame = NULL; this->img_convert_ctx = NULL; this->frame_number = 0; this->decode_frames = true; memset(&(this->rgb_frame), 0, sizeof(this->rgb_frame)); memset(&(this->picture), 0, sizeof(this->picture)); memset(&(this->packet), 0, sizeof(this->packet)); av_init_packet(&(this->packet)); } void VideoCap::release(void) { if (this->img_convert_ctx != NULL) { sws_freeContext(this->img_convert_ctx); this->img_convert_ctx = NULL; } if (this->frame != NULL) { av_frame_free(&(this->frame)); this->frame = NULL; } av_frame_unref(&(this->rgb_frame)); memset(&(this->rgb_frame), 0, sizeof(this->rgb_frame)); memset(&(this->picture), 0, sizeof(this->picture)); if (this->video_dec_ctx != NULL) { avcodec_free_context(&(this->video_dec_ctx)); this->video_dec_ctx = NULL; } if (this->fmt_ctx != NULL) { avformat_close_input(&(this->fmt_ctx)); this->fmt_ctx = NULL; } if (this->opts != NULL) { av_dict_free(&(this->opts)); this->opts = NULL; } if (this->packet.data) { av_packet_unref(&(this->packet)); this->packet.data = NULL; } memset(&packet, 0, sizeof(packet)); av_init_packet(&packet); this->codec = NULL; this->video_stream = NULL; this->video_stream_idx = -1; this->frame_number = 0; this->decode_frames = true; } bool VideoCap::open(const char *url) { bool valid = false; AVStream *st = NULL; int enc_width, enc_height, idx; this->release(); // if another file is already opened if (this->fmt_ctx != NULL) goto error; this->url = url; // open RTSP stream with TCP av_dict_set(&(this->opts), "rtsp_transport", "tcp", 0); av_dict_set(&(this->opts), "stimeout", "5000000", 0); // set timeout to 5 seconds if (avformat_open_input(&(this->fmt_ctx), url, NULL, &(this->opts)) < 0) goto error; // read packets of a media file to get stream information. if (avformat_find_stream_info(this->fmt_ctx, NULL) < 0) goto error; // find the most suitable stream of given type (e.g. video) and set the codec accordingly idx = av_find_best_stream(this->fmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &(this->codec), 0); if (idx < 0) goto error; // set stream in format context this->video_stream_idx = idx; st = this->fmt_ctx->streams[this->video_stream_idx]; // allocate an AVCodecContext and set its fields to default values this->video_dec_ctx = avcodec_alloc_context3(this->codec); if (!this->video_dec_ctx) goto error; // fill the codec context based on the values from the supplied codec parameters if (avcodec_parameters_to_context(this->video_dec_ctx, st->codecpar) < 0) goto error; // ffmpeg recommends no more than 16 threads this->video_dec_ctx->thread_count = std::min(std::thread::hardware_concurrency(), 16u); #ifdef DEBUG std::cerr << "Using parallel processing with " << this->video_dec_ctx->thread_count << " threads" << std::endl; #endif // backup encoder's width/height enc_width = this->video_dec_ctx->width; enc_height = this->video_dec_ctx->height; // Init the video decoder with the codec and set additional option to extract motion vectors av_dict_set(&(this->opts), "flags2", "+export_mvs", 0); if (avcodec_open2(this->video_dec_ctx, this->codec, &(this->opts)) < 0) goto error; this->video_stream = this->fmt_ctx->streams[this->video_stream_idx]; // checking width/height (since decoder can sometimes alter it, eg. vp6f) if (enc_width && (this->video_dec_ctx->width != enc_width)) this->video_dec_ctx->width = enc_width; if (enc_height && (this->video_dec_ctx->height != enc_height)) this->video_dec_ctx->height = enc_height; this->picture.width = this->video_dec_ctx->width; this->picture.height = this->video_dec_ctx->height; this->picture.data = NULL; // print info (duration, bitrate, streams, container, programs, metadata, side data, codec, time base) #ifdef DEBUG av_dump_format(this->fmt_ctx, 0, url, 0); #endif this->frame = av_frame_alloc(); if (!this->frame) goto error; // default: decode frames this->decode_frames = true; if (this->video_stream_idx >= 0) valid = true; error: if (!valid) this->release(); return valid; } void VideoCap::setDecodeFrames(bool enable) { this->decode_frames = enable; } bool VideoCap::getDecodeFrames() { return this->decode_frames; } bool VideoCap::grab(void) { bool valid = false; int got_frame; int count_errs = 0; const int max_number_of_attempts = 512; // make sure file is opened if (!this->fmt_ctx || !this->video_stream) return false; // check if there is a frame left in the stream if (this->fmt_ctx->streams[this->video_stream_idx]->nb_frames > 0 && this->frame_number > this->fmt_ctx->streams[this->video_stream_idx]->nb_frames) return false; // loop over different streams (video, audio) in the file while(!valid) { av_packet_unref(&(this->packet)); // read next packet from the stream int ret = av_read_frame(this->fmt_ctx, &(this->packet)); if (ret == AVERROR(EAGAIN)) continue; // if the packet is not from the video stream don't do anything and get next packet if (this->packet.stream_index != this->video_stream_idx) { av_packet_unref(&(this->packet)); count_errs++; if (count_errs > max_number_of_attempts) break; continue; } // decode the video frame avcodec_decode_video2(this->video_dec_ctx, this->frame, &got_frame, &(this->packet)); if(got_frame) { this->frame_number++; valid = true; } else { count_errs++; if (count_errs > max_number_of_attempts) break; } } return valid; } bool VideoCap::retrieve(uint8_t **frame, int *step, int *width, int *height, int *cn, char *frame_type, MVS_DTYPE **motion_vectors, MVS_DTYPE *num_mvs) { if (!this->video_stream || !(this->frame->data[0])) return false; // perform color conversion and return frame buffer if (this->decode_frames) { if (this->img_convert_ctx == NULL || this->picture.width != this->video_dec_ctx->width || this->picture.height != this->video_dec_ctx->height || this->picture.data == NULL) { // Some sws_scale optimizations have some assumptions about alignment of data/step/width/height // Also we use coded_width/height to workaround problem with legacy ffmpeg versions (like n0.8) int buffer_width = this->video_dec_ctx->coded_width; int buffer_height = this->video_dec_ctx->coded_height; this->img_convert_ctx = sws_getCachedContext( this->img_convert_ctx, buffer_width, buffer_height, this->video_dec_ctx->pix_fmt, buffer_width, buffer_height, AV_PIX_FMT_BGR24, SWS_BICUBIC, NULL, NULL, NULL ); if (this->img_convert_ctx == NULL) return false; av_frame_unref(&(this->rgb_frame)); this->rgb_frame.format = AV_PIX_FMT_BGR24; this->rgb_frame.width = buffer_width; this->rgb_frame.height = buffer_height; if (0 != av_frame_get_buffer(&(this->rgb_frame), 32)) return false; this->picture.width = this->video_dec_ctx->width; this->picture.height = this->video_dec_ctx->height; this->picture.data = this->rgb_frame.data[0]; this->picture.step = this->rgb_frame.linesize[0]; this->picture.cn = 3; } // change color space of frame sws_scale( this->img_convert_ctx, this->frame->data, this->frame->linesize, 0, this->video_dec_ctx->coded_height, this->rgb_frame.data, this->rgb_frame.linesize ); *frame = this->picture.data; *width = this->picture.width; *height = this->picture.height; *step = this->picture.step; *cn = this->picture.cn; } else { // when not decoding frames, don't allocate or return frame buffer *frame = NULL; *width = 0; *height = 0; *step = 0; *cn = 0; } // get motion vectors AVFrameSideData *sd = av_frame_get_side_data(this->frame, AV_FRAME_DATA_MOTION_VECTORS); if (sd) { AVMotionVector *mvs = (AVMotionVector *)sd->data; *num_mvs = sd->size / sizeof(*mvs); if (*num_mvs > 0) { // allocate memory for motion vectors as 1D array if (!(*motion_vectors = (MVS_DTYPE *) malloc(*num_mvs * 10 * sizeof(MVS_DTYPE)))) return false; // store the motion vectors in the allocated memory (C contiguous) for (MVS_DTYPE i = 0; i < *num_mvs; ++i) { *(*motion_vectors + i*10 ) = static_cast(mvs[i].source); *(*motion_vectors + i*10 + 1) = static_cast(mvs[i].w); *(*motion_vectors + i*10 + 2) = static_cast(mvs[i].h); *(*motion_vectors + i*10 + 3) = static_cast(mvs[i].src_x); *(*motion_vectors + i*10 + 4) = static_cast(mvs[i].src_y); *(*motion_vectors + i*10 + 5) = static_cast(mvs[i].dst_x); *(*motion_vectors + i*10 + 6) = static_cast(mvs[i].dst_y); *(*motion_vectors + i*10 + 7) = static_cast(mvs[i].motion_x); *(*motion_vectors + i*10 + 8) = static_cast(mvs[i].motion_y); *(*motion_vectors + i*10 + 9) = static_cast(mvs[i].motion_scale); //*(*motion_vectors + i*11 + 10) = static_cast(mvs[i].flags); } } } // get frame type (I, P, B, etc.) and create a null terminated c-string frame_type[0] = av_get_picture_type_char(this->frame->pict_type); frame_type[1] = '\0'; return true; } bool VideoCap::read(uint8_t **frame, int *step, int *width, int *height, int *cn, char *frame_type, MVS_DTYPE **motion_vectors, MVS_DTYPE *num_mvs) { bool ret = this->grab(); if (ret) ret = this->retrieve(frame, step, width, height, cn, frame_type, motion_vectors, num_mvs); return ret; } ================================================ FILE: src/mvextractor/video_cap.hpp ================================================ #include #include #include #include #include #include // FFMPEG extern "C" { #include #include #include } // for changing the dtype of motion vector #define MVS_DTYPE int32_t #define MVS_DTYPE_NP NPY_INT32 // whether or not to print some debug info //#define DEBUG struct Image_FFMPEG { unsigned char* data; int width; int height; int step; int cn; }; /** * Decode frames and motion vectors from a H264 encoded video file or RTSP stream. * * Implements a VideoCap object similar to OpenCV's VideoCapture. For details * see (https://docs.opencv.org/4.1.0/d8/dfe/classcv_1_1VideoCapture.html). * The class is intended to open a H264 encoded video file or RTSP stream by * providing the according file path or stream url to the `open` method. * Upon sucessful opening of the stream, the `read` method can be called in * a loop each time yielding the next decoded frame of the stream as well as * frame side data, such as motion vectors (as specified per H264 standard). * Instead of calling read, the two methods `grab` and `retrieve` can be used. * `grab` performs reading of the next frame from the stream and decoding which * is fast. `retrieve` performs color space conversion of the frame and motion * vector extraction which is slower. Splitting up `read` like this allows to * generate timestamps which are close to another in case multi-camera setups * are used and captured frames should be close to another. * */ class VideoCap { private: const char *url; AVDictionary *opts; AVCodec *codec; AVFormatContext *fmt_ctx; AVCodecContext *video_dec_ctx; AVStream *video_stream; int video_stream_idx; AVPacket packet; AVFrame *frame; AVFrame rgb_frame; Image_FFMPEG picture; struct SwsContext *img_convert_ctx; int64_t frame_number; // When true, retrieve only motion vectors and skip RGB/color conversion bool decode_frames; #if USE_AV_INTERRUPT_CALLBACK AVInterruptCallbackMetadata interrupt_metadata; #endif public: /** Constructor */ VideoCap(); /** Destroy the VideoCap object and free all ressources */ void release(void); /** Open a video file or RTSP url * * The stream must be H264 encoded. Otherwise, undefined behaviour is * likely. * * @param url Relative or fully specified file path or an RTSP url specifying * the location of the video stream. Example "vid.flv" for a video * file located in the same directory as the source files. Or * "rtsp://xxx.xxx.xxx.xxx:554" for an IP camera streaming via RTSP. * * @retval true if video file or url could be opened sucessfully, false * otherwise. */ bool open(const char *url); /** Reads the next video frame and motion vectors from the stream * * @retval true if a new video frame could be read and decoded, false * otherwise (e.g. at the end of the stream). */ bool grab(void); /** Decodes and returns the grabbed frame and motion vectors * * @param frame Pointer to the raw data of the decoded video frame. The * frame is stored as a C contiguous array of shape (height, width, 3) and * can be converted into a cv::Mat by using the constructor * `cv::Mat cv_frame(height, width, CV_MAKETYPE(CV_8U, 3), frame)`. * Note: A subsequent call of `retrieve` will reuse the same memory for * storing the new frame. If you want a frame to persist for a longer * perdiod of time, allocate a new array and memcopy the raw frame * data into it. After usage you have to manually free this copied * array. * * @param width Width of the returned frame in pixels. * * @param height Height of the returned frame in pixels. * * @param frame_type Either "P", "B" or "I" indicating whether it is an * intra-coded frame (I), a predicted frame with only references to past * frames (P) or reference to both past and future frames (B). Motion * vectors are only returned for "P" and "B" frames. * * @param motion_vectors Pointer to the raw data of the motion vectors * belonging to the decoded frame. The motion vectors are stored as a * C contiguous array of shape (num_mvs, 10). Each row of the array * corresponds to one motion vector. The columns of each vector have the * following meaning (also refer to AVMotionVector in FFMPEG * documentation): * - 0: source: Where the current macroblock comes from. Negative value * when it comes from the past, positive value when it comes * from the future. * - 1: w: Width and height of the vector's macroblock. * - 2: h: Height of the vector's macroblock. * - 3: src_x: x-location of the vector's origin in source frame (in pixels). * - 4: src_y: y-location of the vector's origin in source frame (in pixels). * - 5: dst_x: x-location of the vector's destination in the current frame * (in pixels). * - 6: dst_y: y-location of the vector's destination in the current frame * (in pixels). * - 7: motion_x: src_x = dst_x + motion_x / motion_scale * - 8: motion_y: src_y = dst_y + motion_y / motion_scale * - 9: motion_scale: see definiton of columns 7 and 8 * Note: If no motion vectors are present in a frame, e.g. if the frame is * an I frame, `num_mvs` will be 0 and no memory is allocated for * the motion vectors. * Note: Other than the frame array, new memory for storing motion vectors * is allocated on every call of `retrieve`, thus memcopying is not * needed to persist the motion vectors for a longer period of time. * Note, that the buffer needs to be freed manually by calling * `free(motion_vectors)` when the motion vectors are not needed * anymore. * * @param num_mvs The number of motion vectors corresponding to the rows of * the motion vector array. * * @retval true if the grabbed video frame and motion vectors could be * decoded and returned successfully, false otherwise. */ bool retrieve(uint8_t **frame, int *step, int *width, int *height, int *cn, char *frame_type, MVS_DTYPE **motion_vectors, MVS_DTYPE *num_mvs); /** Convenience wrapper which combines a call of `grab` and `retrieve`. * * The parameters and return value correspond to the `retrieve` method. */ bool read(uint8_t **frame, int *step, int *width, int *height, int *cn, char *frame_type, MVS_DTYPE **motion_vectors, MVS_DTYPE *num_mvs); /** Enable/disable decoding frames in addition to extracting motion vectors. * If decoding is disabled (false), retrieve() will skip color space conversion * and not fill the frame buffer to avoid costly RGB decoding/copying. */ void setDecodeFrames(bool enable); bool getDecodeFrames(); }; ================================================ FILE: tests/README.md ================================================ # Tests ## Run Tests You can run the test suite either directly on your machine or (easier) within the provided Docker container. Both methods require you to first clone the repository. To this end, change into the desired installation directory on your machine and run ```bash git clone https://github.com/LukasBommes/mv-extractor.git mv_extractor ``` ### In Docker Container To run the tests in the Docker container, change into the `mv_extractor` directory, and run ```bash ./run.sh /bin/bash -c 'yum install -y compat-openssl10 && python3.12 -m unittest discover -s tests -p "*tests.py"' ``` ### On Host To run the tests directly on your machine, you need to install the motion vector extractor as explained [above](#step-1-install). Now, change into the `mv_extractor` directory and run the tests with ```bash python3.12 -m unittest discover -s tests -p "*tests.py" ``` Confirm that all tests pass. Some tests run the [LIVE555 Media Server](http://www.live555.com/mediaServer/), which has dependencies on its own, such as OpenSSL. Make sure these dependencies are installed correctly on your machine, or otherwise you will get test failures with messages, such as "error while loading shared libraries: libssl.so.10: cannot open shared object file: No such file or directory". E.g. in Alma Linux you could fix this issue by installing OpenSSL with ```bash yum install -y compat-openssl10 ``` For other operating systems you may be lacking additional dependencies, and the package names and installation command may differ. ## Reference Test Data This directory contains reference test data for validating mv-extractor output. The test suite compares current output against this reference data to ensure no regressions. More specifically the test suite verifies that: 1. Motion vector extraction produces consistent results 2. Frame decoding works correctly 3. Frame types are correctly identified ### Structure - `h264/` - H.264 test video reference data - `mpeg4_part2/` - MPEG-4 Part 2 test video reference data - `rtsp/` - RTSP stream reference data ### Data Format Each subdirectory contains: - `motion_vectors/` - Motion vector .npy files - `frames/` - Frame image .jpg files - `frame_types.txt` - Frame type information ### Reference Data Creation Reference datasets for H.264 and MPEG-4 PART 2 were obtained by running the `extract_mvs` command of a manually verified version of the mvextractor on the provided video files `vid_h264.mp4` and `vid_mpeg4_part2.mp4` RTSP reference data was obtained by streaming one of the video files with the [LIVE555 Media Server](http://www.live555.com/mediaServer/) and then reading the RTSP stream with the motion vector extractor. To reproduce the reference data, follow the steps below. #### Convert input file into H.264 video elementary stream First, convert the `vid_h264.mp4` file into a H.264 video elementary stream file. To this end, run ``` ffmpeg -i vid_h264.mp4 -vf scale=640:360 -vcodec libx264 -f h264 vid_h264.264 ``` in the project's root directory. The conversion is needed, because the LIVE555 Media Server cannot directly serve MP4 files. I also tried converting and serving both input videos as Matroska, which did not work well, and WebM, which did not work at all. Hence, I decided to stick with an H.264 video elementary stream. The command also scales down the input video from 720p to 360p because the default `OutPacketBuffer::maxSize` in the media server is set too low to handle the 720p video. The server logs warnings like ```text MultiFramedRTPSink::afterGettingFrame1(): The input frame data was too large for our buffer size (100176). 10743 bytes of trailing data was dropped! Correct this by increasing "OutPacketBuffer::maxSize" to at least 110743, *before* creating this 'RTPSink'. (Current value is 100000.) ``` and the resulting video frame is truncated at the bottom. #### Serve the video with LIVE555 Media Server Now, we serve the file `vid_h264.264` with LIVE555 Media Server. Place the file in a folder named `data` ``` mkdir -p data cp vid_h264.264 ./data/vid_h264.264 ``` and then run a frash manylinux Docker container, in which you mount the `data` folder as a volume ``` docker run -it -v $(pwd)/data:/data quay.io/pypa/manylinux_2_28_x86_64 /bin/bash ``` In the container install and start the LIVE555 Media Server ``` yum install -y wget compat-openssl10 wget -qP /usr/local/bin/ http://www.live555.com/mediaServer/linux/live555MediaServer chmod +x /usr/local/bin/live555MediaServer cd /data live555MediaServer & ``` You may have to hit `CTRL+C` now to dismiss the log of the server. The server will continue running in the background. #### Consume the RTSP stream with the motion vector extractor Still in the Docker container, install the motion vector extractor ``` python3.12 -m pip install "motion-vector-extractor==1.1.0" ``` and run it to read and dump the RTSP stream to a folder named `out-reference` ``` /opt/python/cp312-cp312/bin/extract_mvs 'rtsp://localhost:554/vid_h264.264' --verbose --dump out-reference ``` #### Preserve reference data and cleanup Finally, exist the container with ``` exit ``` Now, copy the folder `out-reference` into the `tests/reference/rtsp` folder. ``` cp -r data/out-reference tests/reference/rtsp ``` and cleanup ``` rm -rf data ``` ================================================ FILE: tests/end_to_end_tests.py ================================================ import os import time import tempfile import unittest import subprocess import cv2 import numpy as np PROJECT_ROOT = os.getenv("PROJECT_ROOT", "") def motions_vectors_valid(outdir, refdir): equal = [] num_mvs = len(os.listdir(os.path.join(refdir, "motion_vectors"))) for i in range(num_mvs): mvs = np.load(os.path.join(outdir, "motion_vectors", f"mvs-{i}.npy")) mvs_ref = np.load(os.path.join(refdir, "motion_vectors", f"mvs-{i}.npy")) equal.append(np.all(mvs == mvs_ref)) return all(equal) def frame_types_valid(outdir, refdir): with open(os.path.join(outdir, "frame_types.txt"), "r") as file: frame_types = [line.strip() for line in file] with open(os.path.join(refdir, "frame_types.txt"), "r") as file: frame_types_ref = [line.strip() for line in file] return frame_types == frame_types_ref def frames_valid(outdir, refdir): equal = [] num_frames = len(os.listdir(os.path.join(refdir, "frames"))) for i in range(num_frames): frame = cv2.imread(os.path.join(outdir, "frames", f"frame-{i}.jpg")) frame_ref = cv2.imread(os.path.join(refdir, "frames", f"frame-{i}.jpg")) equal.append(np.all(frame == frame_ref)) return all(equal) class TestEndToEnd(unittest.TestCase): def test_end_to_end_h264(self): with tempfile.TemporaryDirectory() as outdir: print("Running extraction for H.264") video_path = os.path.join(PROJECT_ROOT, 'vid_h264.mp4') subprocess.run(f"extract_mvs {video_path} --dump {outdir}", shell=True, check=True) refdir = os.path.join(PROJECT_ROOT, "tests/reference/h264") self.assertTrue(motions_vectors_valid(outdir, refdir), msg="motion vectors are invalid") self.assertTrue(frame_types_valid(outdir, refdir), msg="frame types are invalid") self.assertTrue(frames_valid(outdir, refdir), msg="frames are invalid") def test_end_to_end_motion_vectors_only_h264(self): with tempfile.TemporaryDirectory() as outdir: print("Running motion-vectors-only extraction for H.264") video_path = os.path.join(PROJECT_ROOT, 'vid_h264.mp4') subprocess.run(f"extract_mvs {video_path} --skip-decoding-frames --dump {outdir}", shell=True, check=True) refdir = os.path.join(PROJECT_ROOT, "tests/reference/h264") self.assertTrue(motions_vectors_valid(outdir, refdir), msg="motion vectors are invalid") self.assertTrue(frame_types_valid(outdir, refdir), msg="frame types are invalid") def test_end_to_end_mpeg4_part2(self): with tempfile.TemporaryDirectory() as outdir: print("Running extraction for MPEG-4 Part 2") video_path = os.path.join(PROJECT_ROOT, 'vid_mpeg4_part2.mp4') subprocess.run(f"extract_mvs {video_path} --dump {outdir}", shell=True, check=True) refdir = os.path.join(PROJECT_ROOT, "tests/reference/mpeg4_part2") self.assertTrue(motions_vectors_valid(outdir, refdir), msg="motion vectors are invalid") self.assertTrue(frame_types_valid(outdir, refdir), msg="frame types are invalid") self.assertTrue(frames_valid(outdir, refdir), msg="frames are invalid") def test_end_to_end_motion_vectors_only_mpeg4_part2(self): with tempfile.TemporaryDirectory() as outdir: print("Running motion-vectors-only extraction for MPEG-4 Part 2") video_path = os.path.join(PROJECT_ROOT, 'vid_mpeg4_part2.mp4') subprocess.run(f"extract_mvs {video_path} --skip-decoding-frames --dump {outdir}", shell=True, check=True) refdir = os.path.join(PROJECT_ROOT, "tests/reference/mpeg4_part2") self.assertTrue(motions_vectors_valid(outdir, refdir), msg="motion vectors are invalid") self.assertTrue(frame_types_valid(outdir, refdir), msg="frame types are invalid") def test_end_to_end_rtsp(self): with tempfile.TemporaryDirectory() as outdir: print("Setting up end to end test for RTSP") media_server_binary = os.path.abspath(os.path.join(PROJECT_ROOT, "tests/tools/live555MediaServer")) rtsp_server = subprocess.Popen(media_server_binary, cwd=PROJECT_ROOT if PROJECT_ROOT else None) try: time.sleep(1) print("Running extraction for RTSP stream") rtsp_url = "rtsp://localhost:554/vid_h264.264" subprocess.run(f"extract_mvs {rtsp_url} --dump {outdir}", shell=True, check=True) refdir = os.path.join(PROJECT_ROOT, "tests/reference/rtsp") self.assertTrue(motions_vectors_valid(outdir, refdir), msg="motion vectors are invalid") self.assertTrue(frame_types_valid(outdir, refdir), msg="frame types are invalid") self.assertTrue(frames_valid(outdir, refdir), msg="frames are invalid") finally: rtsp_server.terminate() if __name__ == '__main__': unittest.main() ================================================ FILE: tests/reference/h264/frame_types.txt ================================================ I P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P I P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P P I P P P P P P P P P P P P ================================================ FILE: tests/reference/mpeg4_part2/frame_types.txt ================================================ I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P P P P P P P I P P P P P I P P P P P P P P P P P I ================================================ FILE: tests/reference/rtsp/frame_types.txt ================================================ I B B B P P P P P B B B P B B B P B B B P B B B P B B B P B B B P B B B P B B B P B B B P B B B P B B B P B B B P B B B P B B B P B B B P B B B P B B B P B B B P P P P P P P P B P B P B B B P P P P P P P P P B B P P P P P P P P P P P P P P P P B P P P P P P P P P P P P P B B P P B P P P P P P P P P P P P P P P B P P P P P P P P P P P P P P P P P B P P P P P P B B B P P B B B P B P P P P B B P P P B B B P B B P B B P I B B B P B B B P B B B P B P P P P P B P B P P P P P P P P P P P P P P B P P P P P P P P P B P P B P B P B P B B B P P B B B P B P B P B B B P B B B P B B B P B B P B P B B B P B B B P B B B P B B B P B B B P B B B P B P B P P I B B B P B B P B B B P ================================================ FILE: tests/unit_tests.py ================================================ import os import unittest import time import numpy as np from mvextractor.videocap import VideoCap PROJECT_ROOT = os.getenv("PROJECT_ROOT", "") class TestMotionVectorExtraction(unittest.TestCase): def validate_frame(self, frame): self.assertEqual(type(frame), np.ndarray, "Frame should be numpy array") self.assertEqual(frame.dtype, np.uint8, "Frame dtype should be uint8") self.assertEqual(frame.shape, (720, 1280, 3), "Frams hape should be (720, 1280, 3)") def validate_motion_vectors(self, motion_vectors, shape=(0, 10)): self.assertEqual(type(motion_vectors), np.ndarray, "Motion vectors should be numpy array") self.assertEqual(motion_vectors.dtype, np.int32, "Motion vectors dtype should be int32") self.assertEqual(motion_vectors.shape, shape, "Motion vectors shape not matching expected shape") # run before every test def setUp(self): self.cap = VideoCap() # run after every test regardless of success def tearDown(self): self.cap.release() def open_video(self): return self.cap.open(os.path.join(PROJECT_ROOT, "vid_h264.mp4")) def test_init_cap(self): self.cap = VideoCap() self.assertIn('open', dir(self.cap)) self.assertIn('grab', dir(self.cap)) self.assertIn('read', dir(self.cap)) self.assertIn('release', dir(self.cap)) self.assertIn('retrieve', dir(self.cap)) self.assertIn('set_decode_frames', dir(self.cap)) self.assertIn('decode_frames', dir(self.cap)) def test_decode_frames_mode(self): self.cap = VideoCap() self.assertTrue(self.cap.decode_frames, "Frame decoding is expected to be actived by default") self.cap.set_decode_frames(True) self.assertTrue(self.cap.decode_frames, "Frame decoding is expected to be active") self.cap.set_decode_frames(False) self.assertFalse(self.cap.decode_frames, "Frame decoding is expected to be inactive") self.open_video() self.assertTrue(self.cap.decode_frames, "Frame decoding is expected to be actived after opening a video") self.cap.set_decode_frames(False) self.assertFalse(self.cap.decode_frames, "Frame decoding is expected to be inactive") self.cap.release() self.assertTrue(self.cap.decode_frames, "Frame decoding is expected to be active") def test_open_video(self): ret = self.open_video() self.assertTrue(ret, "Should open video file successfully") def test_open_invalid_video(self): ret = self.cap.open("vid_not_existent.mp4") self.assertFalse(ret, "Should fail to open non-existent video file") def test_read_not_opened_cap(self): ret = self.cap.open("vid_not_existent.mp4") self.assertFalse(ret, "Should fail to open non-existent video file") ret, frame, motion_vectors, frame_type = self.cap.read() self.assertEqual(frame_type, "?", "Frame type should be ?") self.assertFalse(ret, "Should fail to read from non-existent video file") self.assertIsNone(frame, "Frame read from non-existent video should be None") self.validate_motion_vectors(motion_vectors) def test_read_first_I_frame(self): self.open_video() ret, frame, motion_vectors, frame_type = self.cap.read() self.assertTrue(ret, "Should succeed to read from video file") self.assertEqual(frame_type, "I", "Frame type of first frame should be I") self.validate_frame(frame) self.validate_motion_vectors(motion_vectors) def test_read_first_P_frame(self): self.open_video() self.cap.read() # skip first frame (I frame) ret, frame, motion_vectors, frame_type = self.cap.read() self.assertTrue(ret, "Should succeed to read from video file") self.assertEqual(frame_type, "P", "Frame type of second frame should be P") self.validate_frame(frame) self.validate_motion_vectors(motion_vectors, shape=(3665, 10)) self.assertTrue(np.all(motion_vectors[:10, :] == np.array([ [-1, 16, 16, 8, 8, 8, 8, 0, 0, 4], [-1, 16, 16, 24, 8, 24, 8, 0, 0, 4], [-1, 16, 16, 40, 8, 40, 8, 0, 0, 4], [-1, 16, 16, 56, 8, 56, 8, 0, 0, 4], [-1, 16, 16, 72, 8, 72, 8, 0, 0, 4], [-1, 16, 16, 88, 8, 88, 8, 0, 0, 4], [-1, 16, 16, 104, 8, 104, 8, 0, 0, 4], [-1, 16, 16, 120, 8, 120, 8, 0, 0, 4], [-1, 16, 16, 136, 8, 136, 8, 0, 0, 4], [-1, 16, 16, 152, 8, 152, 8, 0, 0, 4], ])), "Motion vectors should match the expected values") def test_read_first_ten_frames(self): rets = [] frames = [] motion_vectors = [] frame_types = [] self.open_video() for _ in range(10): ret, frame, motion_vector, frame_type = self.cap.read() rets.append(ret) frames.append(frame) motion_vectors.append(motion_vector) frame_types.append(frame_type) self.assertTrue(all(rets), "All frames should be read successfully") self.assertEqual(frame_types, ['I', 'P', 'P', 'P', 'P', 'P', 'P', 'P', 'P', 'P']) [self.validate_frame(frame) for frame in frames] shapes = [ (0, 10), (3665, 10), (3696, 10), (3722, 10), (3807, 10), (3953, 10), (4155, 10), (3617, 10), (4115, 10), (4192, 10) ] [self.validate_motion_vectors(motion_vector, shape) for motion_vector, shape in zip(motion_vectors, shapes)] def test_frame_count(self): self.open_video() frame_count = 0 while True: ret, _, _, _ = self.cap.read() if not ret: break frame_count += 1 self.assertEqual(frame_count, 337, "Video file is expected to have 337 frames") def test_timings(self): self.open_video() times = [] while True: tstart = time.perf_counter() ret, _, _, _ = self.cap.read() if not ret: break tend = time.perf_counter() telapsed = tend - tstart times.append(telapsed) dt_mean = np.mean(times) dt_std = np.std(times) print(f"Timings: mean {dt_mean} s -- std: {dt_std} s") self.assertGreater(dt_mean, 0) self.assertGreater(dt_std, 0) self.assertLess(dt_mean, 0.01, msg=f"Mean of frame read duration exceeds maximum ({dt_mean} s > {0.01} s)") self.assertLess(dt_std, 0.003, msg=f"Standard deviation of frame read duration exceeds maximum ({dt_std} s > {0.003} s)") def test_skipping_frame_decoding_does_not_raise(self): self.cap.set_decode_frames(False) self.cap.set_decode_frames(True) def test_read_first_I_frame_skipping_frame_decoding(self): self.open_video() self.cap.set_decode_frames(False) ret, frame, motion_vectors, frame_type = self.cap.read() self.assertTrue(ret, "Should succeed to read from video file") self.assertEqual(frame_type, "I", "Frame type of first frame should be I") self.assertIsNone(frame, "Frame should be None when skipping frame decoding") self.validate_motion_vectors(motion_vectors) def test_read_first_P_frame_skipping_frame_decoding(self): self.open_video() self.cap.set_decode_frames(False) self.cap.read() # skip first frame (I frame) ret, frame, motion_vectors, frame_type = self.cap.read() self.assertTrue(ret, "Should succeed to read from video file") self.assertEqual(frame_type, "P", "Frame type of second frame should be P") self.assertIsNone(frame, "Frame should be None when skipping frame decoding") self.validate_motion_vectors(motion_vectors, shape=(3665, 10)) self.assertTrue(np.all(motion_vectors[:10, :] == np.array([ [-1, 16, 16, 8, 8, 8, 8, 0, 0, 4], [-1, 16, 16, 24, 8, 24, 8, 0, 0, 4], [-1, 16, 16, 40, 8, 40, 8, 0, 0, 4], [-1, 16, 16, 56, 8, 56, 8, 0, 0, 4], [-1, 16, 16, 72, 8, 72, 8, 0, 0, 4], [-1, 16, 16, 88, 8, 88, 8, 0, 0, 4], [-1, 16, 16, 104, 8, 104, 8, 0, 0, 4], [-1, 16, 16, 120, 8, 120, 8, 0, 0, 4], [-1, 16, 16, 136, 8, 136, 8, 0, 0, 4], [-1, 16, 16, 152, 8, 152, 8, 0, 0, 4], ])), "Motion vectors should match the expected values") def test_read_first_ten_frames_skipping_frame_decoding(self): rets = [] frames = [] motion_vectors = [] frame_types = [] self.open_video() self.cap.set_decode_frames(False) for _ in range(10): ret, frame, motion_vector, frame_type = self.cap.read() rets.append(ret) frames.append(frame) motion_vectors.append(motion_vector) frame_types.append(frame_type) self.assertTrue(all(rets), "All frames should be read successfully") self.assertEqual(frame_types, ['I', 'P', 'P', 'P', 'P', 'P', 'P', 'P', 'P', 'P']) [self.assertIsNone(frame) for frame in frames] shapes = [ (0, 10), (3665, 10), (3696, 10), (3722, 10), (3807, 10), (3953, 10), (4155, 10), (3617, 10), (4115, 10), (4192, 10) ] [self.validate_motion_vectors(motion_vector, shape) for motion_vector, shape in zip(motion_vectors, shapes)] def test_frame_count_skipping_frame_decoding(self): self.open_video() self.cap.set_decode_frames(False) frame_count = 0 while True: ret, _, _, _ = self.cap.read() if not ret: break frame_count += 1 self.assertEqual(frame_count, 337, "Video file is expected to have 337 frames") def test_skipping_frame_decoding_is_faster_than_not_skipping(self): self.open_video() # skip frame decoding self.cap.set_decode_frames(False) start_time = time.perf_counter() frame_count = 0 for _ in range(50): # read 50 frames ret, _, _, _ = self.cap.read() if not ret: break frame_count += 1 mvo_time = time.perf_counter() - start_time # do not skip frame decoding self.cap.set_decode_frames(True) start_time = time.perf_counter() frame_count_full = 0 for i in range(50): # Read 50 frames ret, _, _, _ = self.cap.read() if not ret: break frame_count_full += 1 full_time = time.perf_counter() - start_time self.assertEqual(frame_count, 50, "Should read 50 frames") self.assertEqual(frame_count_full, 50, "Should read 50 frames") # Performance comparison (skipping decoding should be at least as fast as not skipping decoding mode) if mvo_time > 0 and full_time > 0: speedup = full_time / mvo_time print(f"Speedup by skipping frame decoding: {speedup:.2f}x") self.assertGreaterEqual(speedup, 1.0, "Skipping frame decoding should be reasonably fast") if __name__ == '__main__': unittest.main()