[ { "path": ".gitignore", "content": "results/*\n**/__pycache__" }, { "path": "Dockerfile", "content": "FROM nvcr.io/nvidia/pytorch:22.10-py3\nENV DEBIAN_FRONTEND=noninteractive\n\nWORKDIR /workspace\n\nRUN apt-get update\nRUN apt-get install rename\nRUN apt-get install -y ffmpeg\n\nCOPY requirements.txt /tmp\nRUN pip install -r /tmp/requirements.txt\n" }, { "path": "LICENSE", "content": "License and Copyright Information\n---------------------------------\nThe contents of this repository may not be used for any purpose other than academic research. It is free to use for research purposes by academic institutes, companies, and individuals. Use for commercial purposes is not permitted without prior written consent from Motorica AB. If you are interested in using the codebase, pretrained models or the dataset for commercial purposes or non-research purposes, please contact us at info@motorica.ai in advance. Unauthorised redistribution is prohibited without written approval.\n\nPlease include the following citations in any preprints and publications that use this repository.\n\n@article{alexanderson2023listen,\n title={Listen, Denoise, Action! Audio-Driven Motion Synthesis with Diffusion Models},\n author={Alexanderson, Simon and Nagy, Rajmund and Beskow, Jonas and Henter, Gustav Eje},\n journal={ACM Trans. Graph.},\n volume={42},\n number={4},\n pages={1--20},\n doi={10.1145/3592458},\n year={2023}\n}\n\nThe code for translation-invariant self-attention (TISA) was written by Ulme Wennberg. Please cite their ACL 2021 article if you use this code.\n" }, { "path": "README.md", "content": "# Listen, Denoise, Action!\nThis repository provides code and models for the paper [Listen, denoise, action! Audio-driven motion synthesis with diffusion models](https://arxiv.org/abs/2211.09707).\n\nPlease watch the following video for an introduction to our work:\n* [SIGGRAPH 2023 presentation](https://youtu.be/Qfd2EpzWgok)\n\nFor video samples and a general overview, please see [our project page](https://www.speech.kth.se/research/listen-denoise-action/). For the new dance dataset with high-quality mocap, please see [the Motorica Dance Dataset](https://github.com/simonalexanderson/MotoricaDanceDataset/).\n\n## Installation\nWe provide a Docker file and `requirements.txt` for installation using a Docker image or Conda.\n\n### Installation using Conda\n```\nconda install python=3.9\nconda install -c conda-forge mpi4py mpich\npip install -r requirements.txt\n```\n\n## Dance synthesis demo\n### Data and pretrained models\nPlease [download our pretrained dance models here](https://zenodo.org/record/8156769) and move them to the `pretrained_models` folder.\nWe include processed music inputs from the test dataset in the `data` folder for generating dances from the model.\n\n### Synthesis scripts\nYou can use the following shell scripts for reproducing the dance user studies in the paper:\n```\n./experiments/dance_LDA.sh\n./experiments/dance_LDA-U.sh\n```\nTo try out locomotion synthesis, please go to https://www.motorica.ai/.\n\n## Training data\nThe four main training datasets from our SIGGRAPH 2023 paper are available online:\n* [The Trinity Speech Gesture Dataset](https://trinityspeechgesture.scss.tcd.ie/)\n* [The ZEGGS dataset](https://github.com/ubisoft/ubisoft-laforge-ZeroEGGS)\n* [The 100STYLE dataset](https://www.ianxmason.com/100style/)\n* [The Motorica Dance Dataset](https://github.com/simonalexanderson/MotoricaDanceDataset/), a new dataset with high-quality dance mocap released together with our paper\n\n## License and copyright information\nThe contents of this repository may not be used for any purpose other than academic research. It is free to use for research purposes by academic institutes, companies, and individuals. Use for commercial purposes is not permitted without prior written consent from Motorica AB. If you are interested in using the codebase, pretrained models or the dataset for commercial purposes or non-research purposes, please contact us at info@motorica.ai in advance. Unauthorised redistribution is prohibited without written approval.\n\n### Attribution\nPlease include the following citations in any preprints and publications that use this repository.\n```\n@article{alexanderson2023listen,\n title={Listen, Denoise, Action! Audio-Driven Motion Synthesis with Diffusion Models},\n author={Alexanderson, Simon and Nagy, Rajmund and Beskow, Jonas and Henter, Gustav Eje},\n year={2023}\n issue_date={August 2023},\n publisher={ACM},\n volume={42},\n number={4},\n doi={10.1145/3592458},\n journal={ACM Trans. Graph.},\n articleno={44},\n numpages={20},\n pages={44:1--44:20}\n}\n```\nThe [code for translation-invariant self-attention](https://github.com/ulmewennberg/tisa) (TISA) was written by [Ulme Wennberg](https://www.kth.se/profile/ulme). Please cite [the correspoding ACL 2021 article](https://aclanthology.org/2021.acl-short.18) if you use this code.\n" }, { "path": "data/motorica_dance/audio18_features.txt", "content": "MFCC_0\nMFCC_1\nMFCC_2\nChroma_0\nChroma_1\nChroma_2\nChroma_3\nChroma_4\nChroma_5\nChroma_6\nChroma_7\nChroma_8\nChroma_9\nChroma_10\nChroma_11\nSpectralflux_0\nBeatactivation_0\nBeat_0\n" }, { "path": "data/motorica_dance/audio29_features.txt", "content": "MFCC_0\nMFCC_1\nMFCC_2\nMFCC_3\nMFCC_4\nMFCC_5\nMFCC_6\nMFCC_7\nMFCC_8\nMFCC_9\nMFCC_10\nMFCC_11\nMFCC_12\nMFCC_13\nMFCC_14\nMFCC_15\nMFCC_16\nMFCC_17\nMFCC_18\nMFCC_19\nChroma_0\nChroma_1\nChroma_2\nChroma_3\nChroma_4\nChroma_5\nSpectralflux_0\nBeatactivation_0\nBeat_0\n" }, { "path": "data/motorica_dance/ch0_spec_beatact_features.txt", "content": "Chroma_0\nSpectralflux_0\nBeatactivation_0\n" }, { "path": "data/motorica_dance/dance_styles.txt", "content": "gCA\ngCH\ngJZ\ngKR\ngLH\ngLO\ngPO\ngTP\n" }, { "path": "data/motorica_dance/dance_test_files.txt", "content": "kthjazz_gCH_sFM_cAll_d02_mCH_ch01_whitemanpaulandhisorchestraloisiana_006_00\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_whitemanpaulandhisorchestraloisiana_006_01\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_bennygoodmansugarfootstomp_003_00\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_bennygoodmansugarfootstomp_003_01\nkthjazz_gTP_sFM_sngl_d02_015_00\nkthjazz_gTP_sFM_sngl_d02_015_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch24_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch24_01\nkthstreet_gKR_sFM_cAll_d01_mKR_ch01_chargedcableupyour_001_00\nkthstreet_gKR_sFM_cAll_d01_mKR_ch01_chargedcableupyour_001_01\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_thisisit_001_00\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_thisisit_001_01\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_lala_001_00\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_lala_001_01\nkthstreet_gLO_sFM_cAll_d02_mLO_ch01_arethafranklinrocksteady_002_00\nkthstreet_gLO_sFM_cAll_d02_mLO_ch01_arethafranklinrocksteady_002_01\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_bombom_002_00\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_bombom_002_01\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_bombom_001_00\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_bombom_001_01\n" }, { "path": "data/motorica_dance/dance_train_files.txt", "content": "kthjazz_gCH_sFM_cAll_d02_mCH_ch01_beatlestreetwashboardbandfortyandtight_003_00\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_beatlestreetwashboardbandfortyandtight_003_00_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_beatlestreetwashboardbandfortyandtight_003_01\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_beatlestreetwashboardbandfortyandtight_003_01_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_charlestonchaserswabashblues_004_00\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_charlestonchaserswabashblues_004_00_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_charlestonchaserswabashblues_004_01\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_charlestonchaserswabashblues_004_01_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_lloydkeatingandhismusicturnontheheat_010_00\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_lloydkeatingandhismusicturnontheheat_010_00_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_lloydkeatingandhismusicturnontheheat_010_01\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_lloydkeatingandhismusicturnontheheat_010_01_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_royalgardenblues_002_00\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_royalgardenblues_002_00_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_royalgardenblues_002_01\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_royalgardenblues_002_01_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_tedlewisgladragdoll_007_00\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_tedlewisgladragdoll_007_00_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_tedlewisgladragdoll_007_01\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_tedlewisgladragdoll_007_01_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_therythmiceightkansascitykitty_001_00\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_therythmiceightkansascitykitty_001_00_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_therythmiceightkansascitykitty_001_01\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_therythmiceightkansascitykitty_001_01_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_therythmiceightumthaumthadadada_009_00\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_therythmiceightumthaumthadadada_009_00_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_therythmiceightumthaumthadadada_009_01\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_therythmiceightumthaumthadadada_009_01_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_thesavoyorpheansfivefoottwoeyesofblue_008_00\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_thesavoyorpheansfivefoottwoeyesofblue_008_00_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_thesavoyorpheansfivefoottwoeyesofblue_008_01\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_thesavoyorpheansfivefoottwoeyesofblue_008_01_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_thesavoyorpheansthecharleston_005_00\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_thesavoyorpheansthecharleston_005_00_mirrored\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_thesavoyorpheansthecharleston_005_01\nkthjazz_gCH_sFM_cAll_d02_mCH_ch01_thesavoyorpheansthecharleston_005_01_mirrored\nkthjazz_gCH_sFM_sngl_d01_007_00\nkthjazz_gCH_sFM_sngl_d01_007_00_mirrored\nkthjazz_gCH_sFM_sngl_d01_007_01\nkthjazz_gCH_sFM_sngl_d01_007_01_mirrored\nkthjazz_gCH_sFM_sngl_d01_008_00\nkthjazz_gCH_sFM_sngl_d01_008_00_mirrored\nkthjazz_gCH_sFM_sngl_d01_008_01\nkthjazz_gCH_sFM_sngl_d01_008_01_mirrored\nkthjazz_gCH_sFM_sngl_d01_009_00\nkthjazz_gCH_sFM_sngl_d01_009_00_mirrored\nkthjazz_gCH_sFM_sngl_d01_009_01\nkthjazz_gCH_sFM_sngl_d01_009_01_mirrored\nkthjazz_gCH_sFM_sngl_d01_019_00\nkthjazz_gCH_sFM_sngl_d01_019_00_mirrored\nkthjazz_gCH_sFM_sngl_d01_019_01\nkthjazz_gCH_sFM_sngl_d01_019_01_mirrored\nkthjazz_gCH_sFM_sngl_d02_005_00\nkthjazz_gCH_sFM_sngl_d02_005_00_mirrored\nkthjazz_gCH_sFM_sngl_d02_005_01\nkthjazz_gCH_sFM_sngl_d02_005_01_mirrored\nkthjazz_gCH_sFM_sngl_d02_006_00\nkthjazz_gCH_sFM_sngl_d02_006_00_mirrored\nkthjazz_gCH_sFM_sngl_d02_006_01\nkthjazz_gCH_sFM_sngl_d02_006_01_mirrored\nkthjazz_gCH_sFM_sngl_d02_018_00\nkthjazz_gCH_sFM_sngl_d02_018_00_mirrored\nkthjazz_gCH_sFM_sngl_d02_018_01\nkthjazz_gCH_sFM_sngl_d02_018_01_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_bennygoodmanairmailspecial_002_00\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_bennygoodmanairmailspecial_002_00_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_bennygoodmanairmailspecial_002_01\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_bennygoodmanairmailspecial_002_01_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_budpowellboundingwithbud_006_00\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_budpowellboundingwithbud_006_00_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_budpowellboundingwithbud_006_01\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_budpowellboundingwithbud_006_01_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_countbasieshortygeorge_007_00\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_countbasieshortygeorge_007_00_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_countbasieshortygeorge_007_01\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_countbasieshortygeorge_007_01_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_dizzygillespiegroovinghigh_004_00\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_dizzygillespiegroovinghigh_004_00_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_dizzygillespiegroovinghigh_004_01\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_dizzygillespiegroovinghigh_004_01_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_dizzygillespieohbopshbam_005_00\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_dizzygillespieohbopshbam_005_00_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_dizzygillespieohbopshbam_005_01\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_dizzygillespieohbopshbam_005_01_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_dukeellingtontaketheatrain_001_00\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_dukeellingtontaketheatrain_001_00_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_dukeellingtontaketheatrain_001_01\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_dukeellingtontaketheatrain_001_01_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_glennmillerinthemood_003_00\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_glennmillerinthemood_003_00_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_glennmillerinthemood_003_01\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_glennmillerinthemood_003_01_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_jimmyluncefordfordancersonly_004_00\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_jimmyluncefordfordancersonly_004_00_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_jimmyluncefordfordancersonly_004_01\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_jimmyluncefordfordancersonly_004_01_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_lionelhamptonflyinghome_005_00\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_lionelhamptonflyinghome_005_00_mirrored\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_lionelhamptonflyinghome_005_01\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_lionelhamptonflyinghome_005_01_mirrored\nkthjazz_gJZ_sFM_sngl_d01_001_00\nkthjazz_gJZ_sFM_sngl_d01_001_00_mirrored\nkthjazz_gJZ_sFM_sngl_d01_001_01\nkthjazz_gJZ_sFM_sngl_d01_001_01_mirrored\nkthjazz_gJZ_sFM_sngl_d01_002_00\nkthjazz_gJZ_sFM_sngl_d01_002_00_mirrored\nkthjazz_gJZ_sFM_sngl_d01_002_01\nkthjazz_gJZ_sFM_sngl_d01_002_01_mirrored\nkthjazz_gJZ_sFM_sngl_d01_020_00\nkthjazz_gJZ_sFM_sngl_d01_020_00_mirrored\nkthjazz_gJZ_sFM_sngl_d01_020_01\nkthjazz_gJZ_sFM_sngl_d01_020_01_mirrored\nkthjazz_gJZ_sFM_sngl_d01_021_00\nkthjazz_gJZ_sFM_sngl_d01_021_00_mirrored\nkthjazz_gJZ_sFM_sngl_d01_021_01\nkthjazz_gJZ_sFM_sngl_d01_021_01_mirrored\nkthjazz_gJZ_sFM_sngl_d02_003_00\nkthjazz_gJZ_sFM_sngl_d02_003_00_mirrored\nkthjazz_gJZ_sFM_sngl_d02_003_01\nkthjazz_gJZ_sFM_sngl_d02_003_01_mirrored\nkthjazz_gJZ_sFM_sngl_d02_004_00\nkthjazz_gJZ_sFM_sngl_d02_004_00_mirrored\nkthjazz_gJZ_sFM_sngl_d02_004_01\nkthjazz_gJZ_sFM_sngl_d02_004_01_mirrored\nkthjazz_gTP_sFM_sngl_d01_010_00\nkthjazz_gTP_sFM_sngl_d01_010_00_mirrored\nkthjazz_gTP_sFM_sngl_d01_010_01\nkthjazz_gTP_sFM_sngl_d01_010_01_mirrored\nkthjazz_gTP_sFM_sngl_d02_014_00\nkthjazz_gTP_sFM_sngl_d02_014_00_mirrored\nkthjazz_gTP_sFM_sngl_d02_014_01\nkthjazz_gTP_sFM_sngl_d02_014_01_mirrored\nkthjazz_gTP_sFM_sngl_d02_016_00\nkthjazz_gTP_sFM_sngl_d02_016_00_mirrored\nkthjazz_gTP_sFM_sngl_d02_016_01\nkthjazz_gTP_sFM_sngl_d02_016_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch0_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch0_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch0_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch0_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch10_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch10_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch10_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch10_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch13_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch13_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch13_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch13_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch14_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch14_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch14_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch14_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch15_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch15_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch15_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch15_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch16_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch16_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch16_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch16_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch17_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch17_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch17_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch17_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch19_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch19_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch19_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch19_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch1_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch1_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch1_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch1_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch20_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch20_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch20_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch20_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch22_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch22_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch22_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch22_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch24_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch24_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch24_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch24_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch28_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch28_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch28_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch28_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch29_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch29_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch29_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch29_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch2_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch2_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch2_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch2_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch30_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch30_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch30_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch30_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch3_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch3_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch3_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch3_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch4_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch4_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch4_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch4_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch5_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch5_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch5_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch5_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch6_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch6_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch6_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch6_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch8_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch8_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch8_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch8_01_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch9_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch9_00_mirrored\nkthmisc_gCA_sFM_cAll_d01_mCA_ch9_01\nkthmisc_gCA_sFM_cAll_d01_mCA_ch9_01_mirrored\nkthstreet_gKR_sFM_cAll_d01_mKR1_ch01_00\nkthstreet_gKR_sFM_cAll_d01_mKR1_ch01_00_mirrored\nkthstreet_gKR_sFM_cAll_d01_mKR1_ch01_01\nkthstreet_gKR_sFM_cAll_d01_mKR1_ch01_01_mirrored\nkthstreet_gKR_sFM_cAll_d01_mKR3_ch03_00\nkthstreet_gKR_sFM_cAll_d01_mKR3_ch03_00_mirrored\nkthstreet_gKR_sFM_cAll_d01_mKR3_ch03_01\nkthstreet_gKR_sFM_cAll_d01_mKR3_ch03_01_mirrored\nkthstreet_gKR_sFM_cAll_d01_mKR4_ch04_00\nkthstreet_gKR_sFM_cAll_d01_mKR4_ch04_00_mirrored\nkthstreet_gKR_sFM_cAll_d01_mKR4_ch04_01\nkthstreet_gKR_sFM_cAll_d01_mKR4_ch04_01_mirrored\nkthstreet_gKR_sFM_cAll_d01_mKR5_ch05_00\nkthstreet_gKR_sFM_cAll_d01_mKR5_ch05_00_mirrored\nkthstreet_gKR_sFM_cAll_d01_mKR5_ch05_01\nkthstreet_gKR_sFM_cAll_d01_mKR5_ch05_01_mirrored\nkthstreet_gKR_sFM_cAll_d01_mKR6_ch06_00\nkthstreet_gKR_sFM_cAll_d01_mKR6_ch06_00_mirrored\nkthstreet_gKR_sFM_cAll_d01_mKR6_ch06_01\nkthstreet_gKR_sFM_cAll_d01_mKR6_ch06_01_mirrored\nkthstreet_gKR_sFM_cAll_d01_mKR7_ch07_00\nkthstreet_gKR_sFM_cAll_d01_mKR7_ch07_00_mirrored\nkthstreet_gKR_sFM_cAll_d01_mKR7_ch07_01\nkthstreet_gKR_sFM_cAll_d01_mKR7_ch07_01_mirrored\nkthstreet_gKR_sFM_cAll_d01_mKR_ch01_1stafterthecaos_000_00\nkthstreet_gKR_sFM_cAll_d01_mKR_ch01_1stafterthecaos_000_00_mirrored\nkthstreet_gKR_sFM_cAll_d01_mKR_ch01_1stafterthecaos_000_01\nkthstreet_gKR_sFM_cAll_d01_mKR_ch01_1stafterthecaos_000_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH11_ch11_00\nkthstreet_gLH_sFM_cAll_d01_mLH11_ch11_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH11_ch11_01\nkthstreet_gLH_sFM_cAll_d01_mLH11_ch11_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH12_ch12_00\nkthstreet_gLH_sFM_cAll_d01_mLH12_ch12_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH12_ch12_01\nkthstreet_gLH_sFM_cAll_d01_mLH12_ch12_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH13_ch13_00\nkthstreet_gLH_sFM_cAll_d01_mLH13_ch13_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH13_ch13_01\nkthstreet_gLH_sFM_cAll_d01_mLH13_ch13_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH14_ch14_00\nkthstreet_gLH_sFM_cAll_d01_mLH14_ch14_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH14_ch14_01\nkthstreet_gLH_sFM_cAll_d01_mLH14_ch14_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH15_ch15_00\nkthstreet_gLH_sFM_cAll_d01_mLH15_ch15_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH15_ch15_01\nkthstreet_gLH_sFM_cAll_d01_mLH15_ch15_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH1_ch01_00\nkthstreet_gLH_sFM_cAll_d01_mLH1_ch01_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH1_ch01_01\nkthstreet_gLH_sFM_cAll_d01_mLH1_ch01_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH2_ch02_00\nkthstreet_gLH_sFM_cAll_d01_mLH2_ch02_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH2_ch02_01\nkthstreet_gLH_sFM_cAll_d01_mLH2_ch02_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH3_ch03_00\nkthstreet_gLH_sFM_cAll_d01_mLH3_ch03_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH3_ch03_01\nkthstreet_gLH_sFM_cAll_d01_mLH3_ch03_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH4_ch04_00\nkthstreet_gLH_sFM_cAll_d01_mLH4_ch04_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH4_ch04_01\nkthstreet_gLH_sFM_cAll_d01_mLH4_ch04_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH6_ch06_00\nkthstreet_gLH_sFM_cAll_d01_mLH6_ch06_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH6_ch06_01\nkthstreet_gLH_sFM_cAll_d01_mLH6_ch06_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH7_ch07_00\nkthstreet_gLH_sFM_cAll_d01_mLH7_ch07_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH7_ch07_01\nkthstreet_gLH_sFM_cAll_d01_mLH7_ch07_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH8_ch08_00\nkthstreet_gLH_sFM_cAll_d01_mLH8_ch08_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH8_ch08_01\nkthstreet_gLH_sFM_cAll_d01_mLH8_ch08_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH9_ch09_00\nkthstreet_gLH_sFM_cAll_d01_mLH9_ch09_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH9_ch09_01\nkthstreet_gLH_sFM_cAll_d01_mLH9_ch09_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_danstrams_00\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_danstrams_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_danstrams_01\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_danstrams_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_feelingsvstech_001_00\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_feelingsvstech_001_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_feelingsvstech_001_01\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_feelingsvstech_001_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_imhere20_005_00\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_imhere20_005_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_imhere20_005_01\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_imhere20_005_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_jesusbreak_001_00\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_jesusbreak_001_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_jesusbreak_001_01\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_jesusbreak_001_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_vibeness108bpm_001_00\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_vibeness108bpm_001_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_vibeness108bpm_001_01\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_vibeness108bpm_001_01_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_yougonnaregretit_003_00\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_yougonnaregretit_003_00_mirrored\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_yougonnaregretit_003_01\nkthstreet_gLH_sFM_cAll_d01_mLH_ch01_yougonnaregretit_003_01_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_basketboll_002_00\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_basketboll_002_00_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_basketboll_002_01\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_basketboll_002_01_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_danstrams_000_00\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_danstrams_000_00_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_danstrams_000_01\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_danstrams_000_01_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_digit_001_00\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_digit_001_00_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_digit_001_01\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_digit_001_01_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_glitter_001_00\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_glitter_001_00_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_glitter_001_01\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_glitter_001_01_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_hitmewobble_001_00\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_hitmewobble_001_00_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_hitmewobble_001_01\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_hitmewobble_001_01_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_luco100bpm_002_00\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_luco100bpm_002_00_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_luco100bpm_002_01\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_luco100bpm_002_01_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_opdirt_002_00\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_opdirt_002_00_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_opdirt_002_01\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_opdirt_002_01_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_thisisit_001_00\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_thisisit_001_00_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_thisisit_001_01\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_thisisit_001_01_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_yougonnaregretit_002_00\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_yougonnaregretit_002_00_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_yougonnaregretit_002_01\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_yougonnaregretit_002_01_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_yourhead108bpm_001_00\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_yourhead108bpm_001_00_mirrored\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_yourhead108bpm_001_01\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_yourhead108bpm_001_01_mirrored\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_boogiewonderland_001_00\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_boogiewonderland_001_00_mirrored\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_boogiewonderland_001_01\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_boogiewonderland_001_01_mirrored\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_dennisscorpiocoffey_001_00\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_dennisscorpiocoffey_001_00_mirrored\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_dennisscorpiocoffey_001_01\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_dennisscorpiocoffey_001_01_mirrored\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_funk_001_00\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_funk_001_00_mirrored\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_funk_001_01\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_funk_001_01_mirrored\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_jamesbrownpapasgotabrandnewbag_001_00\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_jamesbrownpapasgotabrandnewbag_001_00_mirrored\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_jamesbrownpapasgotabrandnewbag_001_01\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_jamesbrownpapasgotabrandnewbag_001_01_mirrored\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_princesexydanser_001_00\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_princesexydanser_001_00_mirrored\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_princesexydanser_001_01\nkthstreet_gLO_sFM_cAll_d01_mLO_ch01_princesexydanser_001_01_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO1_ch01_00\nkthstreet_gPO_sFM_cAll_d01_mPO1_ch01_00_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO1_ch01_01\nkthstreet_gPO_sFM_cAll_d01_mPO1_ch01_01_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO2_ch02_00\nkthstreet_gPO_sFM_cAll_d01_mPO2_ch02_00_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO2_ch02_01\nkthstreet_gPO_sFM_cAll_d01_mPO2_ch02_01_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO3_ch03_00\nkthstreet_gPO_sFM_cAll_d01_mPO3_ch03_00_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO3_ch03_01\nkthstreet_gPO_sFM_cAll_d01_mPO3_ch03_01_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO4_ch04_00\nkthstreet_gPO_sFM_cAll_d01_mPO4_ch04_00_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO4_ch04_01\nkthstreet_gPO_sFM_cAll_d01_mPO4_ch04_01_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO5_ch05_00\nkthstreet_gPO_sFM_cAll_d01_mPO5_ch05_00_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO5_ch05_01\nkthstreet_gPO_sFM_cAll_d01_mPO5_ch05_01_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO6_ch06_00\nkthstreet_gPO_sFM_cAll_d01_mPO6_ch06_00_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO6_ch06_01\nkthstreet_gPO_sFM_cAll_d01_mPO6_ch06_01_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO7_ch07_00\nkthstreet_gPO_sFM_cAll_d01_mPO7_ch07_00_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO7_ch07_01\nkthstreet_gPO_sFM_cAll_d01_mPO7_ch07_01_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_bbonthespot_003_00\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_bbonthespot_003_00_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_bbonthespot_003_01\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_bbonthespot_003_01_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_heropop_002_00\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_heropop_002_00_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_heropop_002_01\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_heropop_002_01_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_justus2_001_00\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_justus2_001_00_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_justus2_001_01\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_justus2_001_01_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_luco100bpm_001_00\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_luco100bpm_001_00_mirrored\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_luco100bpm_001_01\nkthstreet_gPO_sFM_cAll_d01_mPO_ch01_luco100bpm_001_01_mirrored\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_atombounce_001_00\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_atombounce_001_00_mirrored\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_atombounce_001_01\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_atombounce_001_01_mirrored\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_bbonthespot_004_00\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_bbonthespot_004_00_mirrored\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_bbonthespot_004_01\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_bbonthespot_004_01_mirrored\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_funk_002_00\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_funk_002_00_mirrored\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_funk_002_01\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_funk_002_01_mirrored\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_justus2_002_00\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_justus2_002_00_mirrored\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_justus2_002_01\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_justus2_002_01_mirrored\n" }, { "path": "data/motorica_dance/gen_files.txt", "content": "kthjazz_gCH_sFM_cAll_d02_mCH_ch01_whitemanpaulandhisorchestraloisiana_006_00\nkthjazz_gJZ_sFM_cAll_d02_mJZ_ch01_bennygoodmansugarfootstomp_003_00\nkthmisc_gCA_sFM_cAll_d01_mCA_ch24_00\nkthstreet_gKR_sFM_cAll_d01_mKR_ch01_chargedcableupyour_001_00\nkthstreet_gLH_sFM_cAll_d02_mLH_ch01_lala_001_00\nkthstreet_gLO_sFM_cAll_d02_mLO_ch01_arethafranklinrocksteady_002_00\nkthstreet_gPO_sFM_cAll_d02_mPO_ch01_bombom_001_00\n" }, { "path": "data/motorica_dance/pose_features.expmap.txt", "content": "RightFoot_alpha\nRightFoot_beta\nRightFoot_gamma\nRightLeg_alpha\nRightLeg_beta\nRightLeg_gamma\nRightUpLeg_alpha\nRightUpLeg_beta\nRightUpLeg_gamma\nLeftFoot_alpha\nLeftFoot_beta\nLeftFoot_gamma\nLeftLeg_alpha\nLeftLeg_beta\nLeftLeg_gamma\nLeftUpLeg_alpha\nLeftUpLeg_beta\nLeftUpLeg_gamma\nRightHand_alpha\nRightHand_beta\nRightHand_gamma\nRightForeArm_alpha\nRightForeArm_beta\nRightForeArm_gamma\nRightArm_alpha\nRightArm_beta\nRightArm_gamma\nRightShoulder_alpha\nRightShoulder_beta\nRightShoulder_gamma\nLeftHand_alpha\nLeftHand_beta\nLeftHand_gamma\nLeftForeArm_alpha\nLeftForeArm_beta\nLeftForeArm_gamma\nLeftArm_alpha\nLeftArm_beta\nLeftArm_gamma\nLeftShoulder_alpha\nLeftShoulder_beta\nLeftShoulder_gamma\nHead_alpha\nHead_beta\nHead_gamma\nNeck_alpha\nNeck_beta\nNeck_gamma\nSpine1_alpha\nSpine1_beta\nSpine1_gamma\nSpine_alpha\nSpine_beta\nSpine_gamma\nHips_alpha\nHips_beta\nHips_gamma\nHips_Yposition\nreference_dXposition\nreference_dZposition\nreference_dYrotation\n" }, { "path": "experiments/dance_LDA-U.sh", "content": "#!/bin/bash\n\ncheckpoint=pretrained_models/dance_LDA-U.ckpt\ndest_dir=results/generated/dance_LDA-U\n\nif [ ! -d \"${dest_dir}\" ]; then\n mkdir -p \"${dest_dir}\"\nfi\n\ndata_dir=data/motorica_dance\nwav_dir=data/motorica_dance\nbasenames=$(cat \"${data_dir}/gen_files.txt\")\n\nstart=0\nseed=150\nfps=30\ntrim_s=0\nlength_s=10\ntrim=$((trim_s*fps))\nlength=$((length_s*fps))\nfixed_seed=false\ngpu=\"cuda:0\"\nrender_video=true\n\nfor wavfile in $basenames; \ndo\n\tstart=0\n\tfor postfix in 0 1 2 3 4 5 6 7 8 9 10 11\n\tdo\n\t\tinput_file=${wavfile}.audio29_${fps}fps.pkl\n\t\t\n\t\toutput_file=${wavfile::-3}_${postfix}_${style}\n\t\t\n\t\techo \"start=${start}, len=${length}, postfix=${postfix}, seed=${seed}\"\n\t\tpython synthesize.py --checkpoints=\"${checkpoint}\" --data_dirs=\"${data_dir}\" --input_files=\"${input_file}\" --start=${start} --end=${length} --seed=${seed} --postfix=${postfix} --trim=${trim} --dest_dir=${dest_dir} --gpu=${gpu} --video=${render_video} --outfile=${output_file}\n\t\tif [ \"$fixed_seed\" != \"true\" ]; then\n\t\t\tseed=$((seed+1))\n\t\tfi \n\t\techo seed=$seed\n\t\tpython utils/cut_wav.py ${wav_dir}/${wavfile::-3}.wav $(((start+trim)/fps)) $(((start+length-trim)/fps)) ${postfix} ${dest_dir}\n\t\tif [ \"$render_video\" == \"true\" ]; then\n\t\t\tffmpeg -y -i ${dest_dir}/${output_file}.mp4 -i ${dest_dir}/${wavfile::-3}_${postfix}.wav ${dest_dir}/${output_file}_audio.mp4\n\t\t\trm ${dest_dir}/${output_file}.mp4\n\t\tfi\n\t\t\n\t\tstart=$((start+length))\n\tdone\ndone\n" }, { "path": "experiments/dance_LDA.sh", "content": "#!/bin/bash\n\ncheckpoint=pretrained_models/dance_LDA.ckpt\ndest_dir=results/generated/dance_LDA\n\nif [ ! -d \"${dest_dir}\" ]; then\n mkdir -p \"${dest_dir}\"\nfi\n\ndata_dir=data/motorica_dance\nwav_dir=data/motorica_dance\nbasenames=$(cat \"${data_dir}/gen_files.txt\")\n\nstart=0\nseed=150\nfps=30\ntrim_s=0\nlength_s=10\ntrim=$((trim_s*fps))\nlength=$((length_s*fps))\nfixed_seed=false\ngpu=\"cuda:0\"\nrender_video=true\n\nfor wavfile in $basenames; \ndo\n\tstart=0\n\tstyle=$(echo $wavfile | awk -F \"_\" '{print $2}') #Coherent style parsed from file-name\n\tfor postfix in 0 1 2 3 4 5 6 7 8 9 10 11\n\tdo\n\t\tinput_file=${wavfile}.audio29_${fps}fps.pkl\n\t\t\n\t\toutput_file=${wavfile::-3}_${postfix}_${style}\n\t\t\n\t\techo \"start=${start}, len=${length}, postfix=${postfix}, seed=${seed}\"\n\t\tpython synthesize.py --checkpoints=\"${checkpoint}\" --data_dirs=\"${data_dir}\" --input_files=\"${input_file}\" --styles=\"${style}\" --start=${start} --end=${length} --seed=${seed} --postfix=${postfix} --trim=${trim} --dest_dir=${dest_dir} --gpu=${gpu} --video=${render_video} --outfile=${output_file}\n\t\tif [ \"$fixed_seed\" != \"true\" ]; then\n\t\t\tseed=$((seed+1))\n\t\tfi \n\t\techo seed=$seed\n\t\tpython utils/cut_wav.py ${wav_dir}/${wavfile::-3}.wav $(((start+trim)/fps)) $(((start+length-trim)/fps)) ${postfix} ${dest_dir}\n\t\tif [ \"$render_video\" == \"true\" ]; then\n\t\t\tffmpeg -y -i ${dest_dir}/${output_file}.mp4 -i ${dest_dir}/${wavfile::-3}_${postfix}.wav ${dest_dir}/${output_file}_audio.mp4\n\t\t\trm ${dest_dir}/${output_file}.mp4\n\t\tfi\n\t\t\n\t\tstart=$((start+length))\n\tdone\ndone\n" }, { "path": "experiments/dance_mix_experts.sh", "content": "#!/bin/bash\n\n### Mixing N models/conditionings to generate a mix of styles. 1. Model A trained w.o. style conditioning 2. Model B with style S1\n### 3. Model B with style S2. The sampling will be done with e=e0 + g1*(e1-e0) + g2*(e2-e0) + ...\n### gn are guidance factors. E.g. g1=1, g2=0 => style S1 g1=0,g2=1 => S2, g1=1,g2=1 => mix of styles\n\n# # Dance\ncheckpoint1=pretrained_models/dance_LDA-U.ckpt\ncheckpoint2=pretrained_models/dance_LDA.ckpt\ncheckpoint3=${checkpoint2}\ndest_dir=results/dance_mix_experts\n\nif [ ! -d \"${dest_dir}\" ]; then\n mkdir -p \"${dest_dir}\"\nfi\n\ndata_dir=data/motorica_dance\nbasenames=$(cat \"${data_dir}/gen_files.txt\")\n\n# Different guidance factors for mixing models\nguidance_factors_lst=(\"1.0,1.0\" \"0.5,0.5\" \"0.25,1.0\" \"1.0,0.25\")\nstyle=None,gJZ,gLO\n\nstart=0\nseed=150\nfps=30\ntrim_s=0\nlength_s=10\ntrim=$((trim_s*fps))\nlength=$((length_s*fps))\nfixed_seed=false\ngpu=\"cuda:0\"\nrender_video=true\n\nfor wavfile in $basenames;\ndo\n\tstart=0\n\tfor postfix in 0\n\tdo\t\t\n\t\tfor guidance_factors in ${guidance_factors_lst[@]}; do\n\t\n\t\t\tinput_file=${wavfile}.audio29_${fps}fps.pkl\t\t\t\n\t\t\tinput_file2=${input_file}\n\t\t\tinput_file3=${input_file}\t\t\t\t\t\n\t\t\t\n\t\t\toutput_file=${wavfile::-3}_${postfix}_${style}_${guidance_factors}\n\t\t\t\n\t\t\techo Generating motion from ${input_file} to ${dest_dir}/${output_file}\t\t\n\t\t\techo \"start=${start}, len=${length}, postfix=${postfix}, seed=${seed}\"\n\n\t\t\tpython synthesize.py --checkpoints=\"${checkpoint1},${checkpoint2},${checkpoint3}\" --data_dirs=\"${data_dir},${data_dir},${data_dir}\" --input_files=\"${input_file},${input_file2},${input_file3}\" --styles=\"${style}\" --start=${start} --end=${length} --trim=${trim} --seed=${seed} --postfix=${postfix} --dest_dir=${dest_dir} --gf=${guidance_factors} --gpu=${gpu} --outfile=${output_file} --video=${render_video}\n\t\t\tif [ \"$fixed_seed\" != \"true\" ]; then\n\t\t\t\tseed=$((seed+1))\n\t\t\tfi\n\t\t\techo seed=$seed\n\t\t\tpython utils/cut_wav.py ${data_dir}/${wavfile::-3}.wav $(((start+trim)/fps)) $(((start+length-trim)/fps)) ${postfix} ${dest_dir}\n\t\t\tffmpeg -y -i ${dest_dir}/${output_file}.mp4 -i ${dest_dir}/${wavfile::-3}_${postfix}.wav ${dest_dir}/${output_file}_audio.mp4\n\t\t\trm ${dest_dir}/${output_file}.mp4\n\t\t\t\n\t\t\tstart=$((start+length))\n\t\t\tpostfix=$((postfix+1))\n\t\tdone\n\tdone\ndone\n" }, { "path": "hparams/diffusion_dance_LDA-U.yaml", "content": "Data:\n segment_length: 150\n style_index: [1]\n trim_edges: 300\n traindata_filename: dance_train_files_kth.txt\n testdata_filename: dance_test_files_kth.txt\n input_modality: audio35_30fps\n output_modality: expmap_30fps\n input_feats_file: ch0_spec_beatact_features.txt\n datapipe_filename: data_pipe.expmap_30fps.sav\n timestretch_prob: 0.2\n timestretch_factor: 0.1\nDiffusion:\n name: tisa #tisa|conv\n residual_layers: 20\n residual_channels: 256\n embedding_dim: 512\n args:\n tisa:\n num_blocks: 2\n num_heads: 8\n activation: relu\n dropout: 0.1\n norm: LN\n d_ff: 1024\n seq_len: 150 \n use_v2: true\n use_preln: false\n bias: false\n dilation_cycle: [0,1,2]\n conv:\n dilation_cycle_length: 10\n unconditional: false \n noise_schedule_start: 0.01\n noise_schedule_end: 0.7\n n_noise_schedule: 150\nInfer:\n eps: 1\n seq_len: 25\nOptim:\n Schedule:\n args:\n lambda:\n val: 10\n multiplicative:\n val: 10\n step:\n gamma: 0.99995\n step_size: 10\n name: step\n warm_up: 3000\n args:\n adam:\n betas:\n - 0.9\n - 0.999\n eps: 1.0e-08\n rmsprop:\n eps: 1.0e-08\n sgd:\n momentum: 0.9\n name: adam\nValidation:\n render: true\n apply_dropout: false\n render_every_n_epochs: 1\n max_render_clips: 10\n gen_synth_ctrl: false\nlr: 0.0006\nbatch_size: 80\nnum_dataloader_workers: 1\npruning_amount: 0.0\nquantization: false\nTrainer:\n accelerator: gpu\n devices: [2]\n accumulate_grad_batches: 1\n default_root_dir: results/training/dance_LDA-U\n gradient_clip_val: 25\n deterministic: false\n fast_dev_run: false\n max_epochs: 10\n min_epochs: 1\n precision: 32\n resume_from_checkpoint: null \n" }, { "path": "hparams/diffusion_dance_LDA.yaml", "content": "Data:\n segment_length: 150\n style_index: [1]\n trim_edges: 300\n styles_file: dance_styles_kth.txt\n traindata_filename: dance_train_files_kth.txt\n testdata_filename: dance_test_files_kth.txt\n input_modality: audio35_30fps\n output_modality: expmap_30fps\n input_feats_file: ch0_spec_beatact_features.txt\n datapipe_filename: data_pipe.expmap_30fps.sav\n timestretch_prob: 0.2\n timestretch_factor: 0.1\nDiffusion:\n name: tisa #tisa|conv\n residual_layers: 20\n residual_channels: 256\n embedding_dim: 512\n args:\n tisa:\n num_blocks: 2\n num_heads: 8\n activation: relu\n dropout: 0.1\n norm: LN\n d_ff: 1024\n seq_len: 150 \n use_preln: false\n bias: false\n dilation_cycle: [0,1,2]\n conv:\n dilation_cycle_length: 10\n unconditional: false \n noise_schedule_start: 0.01\n noise_schedule_end: 0.7\n n_noise_schedule: 150\nInfer:\n eps: 1\n seq_len: 25\nOptim:\n Schedule:\n args:\n lambda:\n val: 10\n multiplicative:\n val: 10\n step:\n gamma: 0.99995\n step_size: 10\n name: step\n warm_up: 3000\n args:\n adam:\n betas:\n - 0.9\n - 0.999\n eps: 1.0e-08\n rmsprop:\n eps: 1.0e-08\n sgd:\n momentum: 0.9\n name: adam\nValidation:\n render: true\n apply_dropout: false\n render_every_n_epochs: 1\n max_render_clips: 10\n gen_synth_ctrl: false\nlr: 0.0006\nbatch_size: 80\nnum_dataloader_workers: 1\npruning_amount: 0.0\nquantization: false\nTrainer:\n accelerator: gpu\n devices: [2]\n accumulate_grad_batches: 1\n default_root_dir: results/training/dance_LDA\n gradient_clip_val: 25\n deterministic: false\n fast_dev_run: false\n max_epochs: 10\n min_epochs: 1\n precision: 32\n resume_from_checkpoint: null \n" }, { "path": "models/BaseModel.py", "content": "# Copyright 2023 Motorica AB, Inc. All Rights Reserved.\n\nimport os\n\nimport torch\nimport numpy as np\nfrom torch.optim import SGD, Adam, RMSprop\nfrom torch.optim.lr_scheduler import LambdaLR, MultiplicativeLR, StepLR, CosineAnnealingWarmRestarts\nfrom pytorch_lightning import LightningModule\nfrom utils.logging_mixin import LoggingMixin\n\nfrom typing import Tuple, Optional, Union, Dict\nfrom argparse import Namespace\n\nclass BaseModel(LoggingMixin, LightningModule):\n def __init__(self, conf: Optional[Union[Dict, Namespace]] = None, **kwargs):\n super().__init__()\n \n self.save_hyperparameters(conf)\n \n scalers = self.hparams.Data[\"scalers\"]\n #Get input and output scalers from hparams\n input_means=np.array([])\n input_stds=np.array([])\n if scalers[\"in_scaler\"] is not None:\n input_means = scalers[\"in_scaler\"].mean_\n input_stds = scalers[\"in_scaler\"].scale_ \n\n self.input_means = torch.from_numpy(input_means)\n self.input_scales = torch.from_numpy(input_stds)\n self.output_means = torch.from_numpy(scalers[\"out_scaler\"].mean_)\n self.output_scales = torch.from_numpy(scalers[\"out_scaler\"].scale_)\n \n def get_scalers(self):\n return self.hparams.Data[\"scalers\"]\n \n # standarize input\n def standardizeInput(self, input_tensor):\n return ((input_tensor - self.input_means.type_as(input_tensor)) / self.input_scales.type_as(input_tensor))\n\n # standarize output\n def standardizeOutput(self, output_tensor):\n return ((output_tensor - self.output_means.type_as(output_tensor)) / self.output_scales.type_as(output_tensor))\n\n # Add scale and means to output\n def destandardizeInput(self, input_tensor):\n return (input_tensor * self.input_scales.type_as(input_tensor) + self.input_means.type_as(input_tensor))\n\n # Add scale and means to output\n def destandardizeOutput(self, predictions):\n return (predictions * self.output_scales.type_as(predictions) + self.output_means.type_as(predictions))\n \n def configure_optimizers(self):\n lr_params = self.hparams.Optim\n optim_args = lr_params[\"args\"][lr_params[\"name\"]]\n optimizers = {\"adam\": Adam, \"sgd\": SGD, \"rmsprop\": RMSprop}\n # Define optimizer\n optimizer = optimizers[lr_params[\"name\"]](\n self.parameters(), lr=self.hparams.lr, **optim_args\n )\n\n # Define Learning Rate Scheduling\n def lambda1(val):\n return lambda epoch: epoch // val\n\n sched_params = self.hparams.Optim[\"Schedule\"]\n sched_name = sched_params[\"name\"]\n if not sched_name:\n return optimizer\n\n sched_args = sched_params[\"args\"][sched_name]\n\n if sched_name == \"step\":\n scheduler = StepLR(optimizer, **sched_args)\n elif sched_name == \"multiplicative\":\n scheduler = MultiplicativeLR(\n optimizer, lr_lambda=[lambda1(sched_args[\"val\"])]\n )\n elif sched_name == \"lambda\":\n scheduler = LambdaLR(optimizer, lr_lambda=[lambda1(sched_args[\"val\"])])\n elif sched_name == \"cos_warm\":\n scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=sched_args[\"T_0\"])\n else:\n raise NotImplementedError(\"Unimplemented Scheduler!\")\n\n #return [optimizer], [scheduler]\n return {\n \"optimizer\": optimizer,\n \"lr_scheduler\": {\n \"scheduler\": scheduler,\n \"interval\": \"step\",\n },\n }\n\n # learning rate warm-up\n def on_before_optimizer_step(self, optimizer, optimizer_idx):\n lr = self.hparams.lr\n #warm up lr\n warm_up = self.hparams.Optim[\"Schedule\"][\"warm_up\"]\n if self.trainer.global_step < warm_up:\n lr_scale = min(1.0, float(self.trainer.global_step + 1) / warm_up)\n lr *= lr_scale\n for pg in optimizer.param_groups:\n pg[\"lr\"] = lr\n \n\n\n" }, { "path": "models/LightningModel.py", "content": "# Copyright 2023 Motorica AB, Inc. All Rights Reserved.\n\nimport os\nimport sys\n\nimport numpy as np\nimport torch\nfrom torch import nn, Tensor\nfrom torch.nn import functional as F\nfrom models.BaseModel import BaseModel\nfrom models.nn import LDA\n\nfrom typing import Tuple, Optional, Union, Dict\nfrom argparse import Namespace\n\nclass LitLDA(BaseModel):\n def __init__(self, conf, **kwargs):\n super().__init__(conf)\n\n self.input_dim = 0\n self.style_dim = 0\n if self.hparams.Data[\"scalers\"][\"in_scaler\"] is not None:\n self.input_dim = self.hparams.Data[\"scalers\"][\"in_scaler\"].mean_.shape[0]\n if self.hparams.Data[\"scalers\"][\"style_scaler\"] is not None:\n self.style_dim = self.hparams.Data[\"scalers\"][\"style_scaler\"].mean_.shape[0]\n self.pose_dim = self.hparams.Data[\"scalers\"][\"out_scaler\"].mean_.shape[0]\n self.g_cond_dim = self.style_dim\n self.unconditional = self.input_dim == 0\n n_timesteps = self.hparams.Data[\"segment_length\"]\n diff_params = self.hparams.Diffusion\n \n beta_min = diff_params[\"noise_schedule_start\"]\n beta_max = diff_params[\"noise_schedule_end\"]\n self.n_noise_schedule = diff_params[\"n_noise_schedule\"]\n \n self.noise_schedule_name = \"linear\" \n self.noise_schedule = torch.linspace(beta_min, beta_max, self.n_noise_schedule)\n self.noise_level = torch.cumprod(1 - self.noise_schedule, dim=0)\n \n nn_name = diff_params[\"name\"]\n nn_args = diff_params[\"args\"][nn_name]\n \n self.diffusion_model = LDA(self.pose_dim, \n self.hparams.Diffusion[\"residual_layers\"],\n self.hparams.Diffusion[\"residual_channels\"],\n self.hparams.Diffusion[\"embedding_dim\"],\n self.input_dim,\n self.g_cond_dim,\n self.n_noise_schedule,\n nn_name,\n nn_args)\n \n self.loss_fn = nn.MSELoss()\n \n \n def get_input_dim(self):\n return self.input_dim\n \n def get_style_dim(self):\n return self.style_dim\n \n def get_pose_dim(self):\n return self.pose_dim\n \n def diffusion(self, poses, t):\n N, T, C = poses.shape\n noise = torch.randn_like(poses)\n noise_scale = self.noise_level.type_as(noise)[t].unsqueeze(1).unsqueeze(2).repeat(1,T,C)\n noise_scale_sqrt = noise_scale**0.5\n noisy_poses = noise_scale_sqrt * poses + (1.0 - noise_scale)**0.5 * noise\n return noisy_poses, noise\n \n def forward(self, batch):\n\n ctrl, global_cond, poses =batch\n \n N, T, C = poses.shape\n\n num_noisesteps = self.n_noise_schedule\n t = torch.randint(0, num_noisesteps, [N], device=poses.device)\n\n noise = torch.randn_like(poses)\n noise_scale = self.noise_level.type_as(noise)[t].unsqueeze(1).unsqueeze(2).repeat(1,T,C)\n noise_scale_sqrt = noise_scale**0.5\n noisy_poses = noise_scale_sqrt * poses + (1.0 - noise_scale)**0.5 * noise\n noisy_poses, noise = self.diffusion(poses, t)\n predicted = self.diffusion_model(noisy_poses, ctrl, global_cond, t)\n\n loss = self.loss_fn(noise, predicted.squeeze(1))\n \n return loss\n\n def training_step(self, batch, batch_idx):\n loss = self(batch)\n self.log('Loss/train', loss, sync_dist=True)\n\n return loss\n\n def validation_step(self, batch, batch_idx):\n\n loss = self(batch)\n\n self.log('val_loss', loss, prog_bar=True, sync_dist=True)\n\n if (self.trainer.global_step > 0 and\n batch_idx==0 and\n (self.trainer.current_epoch == self.trainer.max_epochs-1 or self.trainer.current_epoch % self.hparams.Validation[\"render_every_n_epochs\"]==0)):\n\n # Log results for the validation data\n self.synthesize_and_log(batch, \"val\")\n\n \n output = {\"val_loss\": loss}\n return output\n\n def validation_epoch_end(self, outputs):\n avg_loss = torch.stack([x[\"val_loss\"] for x in outputs]).mean()\n self.log('Loss/val', avg_loss, sync_dist=True)\n\n def synthesize_and_log(self, batch, log_prefix):\n ctrl, g_cond, _ =batch\n clips = self.synthesize(ctrl, g_cond)\n\n self.log_jerk(clips[:,:,:self.pose_dim], log_prefix)\n file_name = f\"{self.current_epoch}_{self.global_step}_{log_prefix}\"\n self.log_results(clips.cpu().detach().numpy(), file_name, log_prefix, render_video=False)\n\n def test_step(self, batch, batch_idx):\n loss = self(batch)\n\n self.synthesize_and_log(batch, \"test\")\n\n output = {\"test_loss\": loss}\n return output\n \n def synthesize(self, ctrl, global_cond):\n print(\"synthesize\")\n\n training_noise_schedule = self.noise_schedule.to(ctrl.device)\n inference_noise_schedule = training_noise_schedule\n\n talpha = 1 - training_noise_schedule\n talpha_cum = torch.cumprod(talpha, dim=0)\n\n beta = inference_noise_schedule\n alpha = 1 - beta\n alpha_cum = torch.cumprod(alpha, dim=0)\n\n T = []\n for s in range(len(inference_noise_schedule)):\n for t in range(len(training_noise_schedule) - 1):\n if talpha_cum[t+1] <= alpha_cum[s] <= talpha_cum[t]:\n twiddle = (talpha_cum[t]**0.5 - alpha_cum[s]**0.5) / (talpha_cum[t]**0.5 - talpha_cum[t+1]**0.5)\n T.append(t + twiddle)\n break\n \n\n if len(ctrl.shape) == 2:# Expand rank 2 tensors by adding a batch dimension.\n ctrl = ctrl.unsqueeze(0)\n global_cond = global_cond.unsqueeze(0)\n poses = torch.randn(ctrl.shape[0], ctrl.shape[1], self.pose_dim, device=ctrl.device)\n \n nbatch = poses.size(0)\n noise_scale = (alpha_cum**0.5).type_as(poses).unsqueeze(1)\n\n for n in range(len(alpha) - 1, -1, -1):\n c1 = 1 / alpha[n]**0.5\n c2 = beta[n] / (1 - alpha_cum[n])**0.5\n \n poses = c1 * (poses - c2 * self.diffusion_model(poses, ctrl, global_cond, T[n].unsqueeze(-1)).squeeze(1))\n\n if n > 0:\n noise = torch.randn_like(poses)\n sigma = ((1.0 - alpha_cum[n-1]) / (1.0 - alpha_cum[n]) * beta[n])**0.5\n poses += sigma * noise\n \n anim_clip = self.destandardizeOutput(poses)\n if not self.unconditional:\n out_ctrl = self.destandardizeInput(ctrl)\n anim_clip = torch.cat((anim_clip, out_ctrl), dim=2) \n\n return anim_clip\n \n" }, { "path": "models/nn.py", "content": "# Copyright 2023 Motorica AB, Inc. All Rights Reserved.\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom models.transformer.tisa_transformer import TisaTransformer\nfrom math import sqrt\n\nclass Conv1dLayer(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, padding=0, dilation=1):\n super().__init__()\n self.conv1d = nn.Conv1d(in_channels, out_channels, kernel_size=kernel_size, padding=padding, dilation=dilation)\n nn.init.kaiming_normal_(self.conv1d.weight)\n\n def forward(self, x):\n return self.conv1d(x.permute(0,2,1)).permute(0,2,1)\n\ndef silu(x):\n return x * torch.sigmoid(x)\n\nclass DiffusionEmbedding(nn.Module):\n def __init__(self, max_steps, in_channels, hidden_channels):\n super().__init__()\n \n self.in_channels = in_channels\n self.register_buffer('embedding', self._build_embedding(max_steps), persistent=False)\n self.projection1 = nn.Linear(in_channels, hidden_channels)\n self.projection2 = nn.Linear(hidden_channels, hidden_channels)\n\n def forward(self, diffusion_step):\n if diffusion_step.dtype in [torch.int32, torch.int64]:\n x = self.embedding[diffusion_step]\n else:\n x = self._lerp_embedding(diffusion_step)\n x = self.projection1(x)\n x = silu(x)\n x = self.projection2(x)\n x = silu(x)\n return x\n\n def _lerp_embedding(self, t):\n low_idx = torch.floor(t).long()\n high_idx = torch.ceil(t).long()\n low = self.embedding[low_idx]\n high = self.embedding[high_idx]\n return low + (high - low) * (t - low_idx)\n\n def _build_embedding(self, max_steps):\n steps = torch.arange(max_steps).unsqueeze(1) # [T,1]\n dims = torch.arange(64).unsqueeze(0) # [1,64]\n table = steps * 10.0**(dims * 4.0 / 63.0) # [T,64]\n table = torch.cat([torch.sin(table), torch.cos(table)], dim=1)\n return table\n \nclass ResidualBlock(nn.Module):\n def __init__(self, residual_channels, \n embedding_dim, \n l_cond_dim,\n nn_name,\n nn_args,\n index):\n\n super().__init__()\n if nn_name==\"tisa\":\n dilation_cycle = nn_args[\"dilation_cycle\"]\n dilation=dilation_cycle[(index % len(dilation_cycle))]\n self.nn = TisaTransformer(residual_channels, 2 * residual_channels, d_model=residual_channels, num_blocks=nn_args[\"num_blocks\"], num_heads=nn_args[\"num_heads\"], activation=nn_args[\"activation\"], norm=nn_args[\"norm\"], drop_prob=nn_args[\"dropout\"], d_ff=nn_args[\"d_ff\"], seqlen=nn_args[\"seq_len\"], use_preln=nn_args[\"use_preln\"], bias=nn_args[\"bias\"], dilation=dilation)\n elif nn_name==\"conv\":\n dilation=2**(index % nn_args[\"dilation_cycle_length\"])\n self.nn = Conv1dLayer(residual_channels, 2 * residual_channels, 3, padding=dilation, dilation=dilation)\n else:\n raise ValueError(f\"Unknown nn_name: {nn_name}\")\n \n self.l_cond_dim = l_cond_dim\n \n self.diffusion_projection = nn.Linear(embedding_dim, residual_channels)\n self.local_cond_projection = nn.Linear(l_cond_dim, residual_channels)\n self.output_projection = Conv1dLayer(residual_channels, 2 * residual_channels, 1)\n self.residual_channels = residual_channels\n\n def forward(self, x, diffusion_step, local_cond):\n diffusion_step = self.diffusion_projection(diffusion_step).unsqueeze(1)\n y = x + diffusion_step\n\n if self.l_cond_dim > 0:\n y += self.local_cond_projection(local_cond)\n y = self.nn(y).squeeze(-1)\n\n gate, filter = torch.chunk(y, 2, dim=2)\n y = torch.sigmoid(gate) * torch.tanh(filter)\n\n y = self.output_projection(y)\n residual, skip = torch.chunk(y, 2, dim=2)\n return (x + residual) / sqrt(2.0), skip\n\n\nclass LDA(nn.Module):\n def __init__(self,\n pose_dim,\n residual_layers,\n residual_channels,\n embedding_dim,\n l_cond_dim,\n g_cond_dim,\n n_noise_schedule,\n nn_name,\n nn_args):\n super().__init__()\n self.input_projection = Conv1dLayer(pose_dim, residual_channels, 1)\n self.diffusion_embedding = DiffusionEmbedding(n_noise_schedule, 128, embedding_dim)\n\n self.residual_layers = nn.ModuleList([\n ResidualBlock(residual_channels,\n embedding_dim,\n l_cond_dim + g_cond_dim,\n nn_name,\n nn_args,\n i)\n for i in range(residual_layers)\n ])\n self.skip_projection = Conv1dLayer(residual_channels, residual_channels, 1)\n self.output_projection = Conv1dLayer(residual_channels, pose_dim, 1)\n nn.init.zeros_(self.output_projection.conv1d.weight)\n self.l_cond_dim = l_cond_dim\n self.g_cond_dim = g_cond_dim\n\n def forward(self, x, local_cond, global_cond, diffusion_step):\n x = self.input_projection(x)\n x = F.relu(x)\n \n diffusion_step = self.diffusion_embedding(diffusion_step)\n if self.g_cond_dim > 0:\n local_cond=torch.cat((local_cond, global_cond), dim=2)\n\n skip = None\n i=1\n for layer in self.residual_layers:\n x, skip_connection = layer(x, diffusion_step, local_cond)\n skip = skip_connection if skip is None else skip_connection + skip\n \n if skip is not None:\n x = skip / sqrt(len(self.residual_layers))\n x = self.skip_projection(x)\n x = F.relu(x)\n x = self.output_projection(x)\n return x\n" }, { "path": "models/transformer/tisa_transformer.py", "content": "# Copyright 2023 Motorica AB, Inc. All Rights Reserved.\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom models.transformer.tisa_v2 import TisaV2\n\nclass TisaTransformer(nn.Module):\n\n def __init__(\n self,\n in_channels,\n out_channels,\n d_model,\n num_blocks,\n num_heads,\n activation,\n norm,\n drop_prob,\n d_ff=2048,\n tisa_num_kernels=21,\n seqlen=128,\n use_preln=False,\n bias=False,\n dilation=1\n ):\n super(TisaTransformer, self).__init__()\n self.in_proj = nn.Linear(in_channels, d_model)\n self.attention_blocks = nn.ModuleList(\n [\n AttnBlock(d_model, num_heads, activation, norm, drop_prob, tisa_num_kernels, seqlen, use_preln, d_ff, bias=bias, dilation=dilation)\n for _ in range(num_blocks)\n ]\n )\n self.out_proj = nn.Linear(d_model, out_channels)\n\n def forward(self, x):\n x = self.in_proj(x)\n for layer in self.attention_blocks:\n x = layer(x)\n x = self.out_proj(x)\n return x\n\nclass Norm(nn.Module):\n def __init__(self, d_model, eps = 1e-6):\n super().__init__()\n \n self.size = d_model\n # create two learnable parameters to calibrate normalisation\n self.alpha = nn.Parameter(torch.ones(self.size))\n self.bias = nn.Parameter(torch.zeros(self.size))\n self.eps = eps\n def forward(self, x):\n norm = self.alpha * (x - x.mean(dim=-1, keepdim=True)) \\\n / (x.std(dim=-1, keepdim=True) + self.eps) + self.bias\n return norm\n \nclass AttnBlock(nn.Module):\n def __init__(self, d_model, num_heads, activation, norm, drop_prob, tisa_num_kernels, seqlen, use_preln=False, d_ff=2048, bias=False, dilation=1):\n super(AttnBlock, self).__init__()\n self.use_preln = use_preln\n self.attn = GatedAttn(d_model, num_heads=num_heads, activation=activation, seqlen=seqlen, drop_prob=drop_prob, tisa_num_kernels=tisa_num_kernels)\n if (dilation>0):\n self.ff = ConvLayer(d_model, d_ff, activation=activation, dilation=dilation, dropout=drop_prob, bias=bias)\n else:\n self.ff = FeedForward(d_model, d_ff, activation=activation, dropout=drop_prob, bias=bias)\n if norm == \"T5\":\n self.norm_1 = T5LayerNorm(d_model)\n self.norm_2 = T5LayerNorm(d_model)\n elif norm == \"LN\":\n self.norm_1 = nn.LayerNorm(d_model)\n self.norm_2 = nn.LayerNorm(d_model)\n else:\n raise ValueError(f\"unknown norm: {norm}\")\n \n self.dropout_1 = nn.Dropout(drop_prob)\n self.dropout_2 = nn.Dropout(drop_prob)\n\n def forward(self, x):\n if self.use_preln:\n x = self.dropout_1(self.attn(self.norm_2(x))) + x\n else:\n x = self.dropout_1(self.attn(x)) + x\n x = self.norm_2(x)\n\n if self.use_preln:\n x = self.dropout_2(self.ff(self.norm_1(x))) + x\n else:\n x = self.dropout_2(self.ff(x)) + x\n x = self.norm_1(x)\n\n return x\n \nclass T5LayerNorm(nn.Module):\n def __init__(self, hidden_size, eps=1e-6):\n \"\"\"\n Construct a layernorm module in the T5 style. No bias and no subtraction of mean.\n \"\"\"\n super().__init__()\n self.weight = nn.Parameter(torch.ones(hidden_size))\n self.variance_epsilon = eps\n\n def forward(self, hidden_states):\n\n # T5 uses a layer_norm which only scales and doesn't shift, which is also known as Root Mean\n # Square Layer Normalization https://arxiv.org/abs/1910.07467 thus varience is calculated\n # w/o mean and there is no bias. Additionally we want to make sure that the accumulation for\n # half-precision inputs is done in fp32\n\n variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)\n hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)\n\n return self.weight * hidden_states\n \nclass ConvLayer(nn.Module):\n def __init__(self, d_model, d_ff=2048, activation=\"relu\", dilation=1, dropout = 0.1, bias=False):\n super().__init__() \n # We set d_ff as a default to 2048\n self.conv_1 = nn.Conv1d(d_model, d_ff, 3, padding=dilation, dilation=dilation, bias=bias)\n self.dropout = nn.Dropout(dropout)\n self.conv_2 = nn.Conv1d(d_ff, d_model, 3, padding=dilation, dilation=dilation, bias=bias)\n if activation==\"relu\":\n self.act = nn.ReLU()\n elif activation==\"gelu\":\n self.act = nn.GELU()\n def forward(self, x):\n x = self.dropout(self.act(self.conv_1(x.permute(0,2,1))))\n x = self.conv_2(x)\n return x.permute(0,2,1)\n \nclass FeedForward(nn.Module):\n def __init__(self, d_model, d_ff=2048, activation=\"RELU\", dropout = 0.1, bias=False):\n super().__init__() \n # We set d_ff as a default to 2048\n self.linear_1 = nn.Linear(d_model, d_ff, bias=bias)\n self.dropout = nn.Dropout(dropout)\n self.linear_2 = nn.Linear(d_ff, d_model, bias=bias)\n if activation==\"relu\":\n self.act = nn.ReLU()\n elif activation==\"gelu\":\n self.act = nn.GELU()\n def forward(self, x):\n x = self.dropout(self.act(self.linear_1(x)))\n x = self.linear_2(x)\n return x\n \nclass GatedAttn(nn.Module):\n \"\"\"Gated Multi-Head Self-Attention Block\n\n Based on the paper:\n \"Attention Is All You Need\"\n by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones,\n Aidan N. Gomez, Lukasz Kaiser, Illia Polosukhin\n (https://arxiv.org/abs/1706.03762).\n\n Args:\n d_model (int): Number of channels in the input.\n num_heads (int): Number of attention heads.\n drop_prob (float): Dropout probability.\n \"\"\"\n\n def __init__(self, d_model, num_heads=4, activation=\"RELU\", seqlen=128, drop_prob=0.0, tisa_num_kernels=21):\n super(GatedAttn, self).__init__()\n\n assert d_model%num_heads==0, f\"num_heads ({num_heads}) is not evenly divisible by d_model ({d_model})\"\n \n self.d_model = d_model\n self.num_heads = num_heads\n if drop_prob>0:\n self.dropout = nn.Dropout(drop_prob)\n else:\n self.dropout = None\n \n self.in_proj = nn.Linear(d_model, 3 * d_model, bias=False)\n self.gate = nn.Linear(d_model, 2 * d_model)\n self.key_depth_per_head = torch.tensor(self.d_model / self.num_heads)\n\n self.position_scorer = TisaV2(self.num_heads, \n tisa_num_kernels=tisa_num_kernels, \n tisa_dropout_prob=drop_prob, \n num_position_agnostic_heads=0, \n max_field_view=seqlen//2, \n min_field_view=5)\n \n self.position_scorer._init_weights()\n \n\n def forward(self, x):\n b, h, c = x.size()\n _, seq_len, num_channels = x.size()\n\n # Compute q, k, v\n memory, query = torch.split(self.in_proj(x), (2 * c, c), dim=-1)\n q = self.split_last_dim(query, self.num_heads)\n k, v = [\n self.split_last_dim(tensor, self.num_heads)\n for tensor in torch.split(memory, self.d_model, dim=2)\n ]\n\n # Compute attention and reshape\n x = self.dot_product_attention(q, k, v)\n \n x = self.combine_last_two_dim(x.permute(0, 2, 1, 3))\n \n \n x = self.gate(x)\n a, b = x.chunk(2, dim=-1)\n x = a * torch.sigmoid(b)\n\n return x\n\n def dot_product_attention(self, q, k, v):\n \"\"\"Dot-product attention.\n\n Args:\n q (torch.Tensor): Queries of shape (batch, heads, length_q, depth_k)\n k (torch.Tensor): Keys of shape (batch, heads, length_kv, depth_k)\n v (torch.Tensor): Values of shape (batch, heads, length_kv, depth_v)\n bias (bool): Use bias for attention.\n\n Returns:\n attn (torch.Tensor): Output of attention mechanism.\n \"\"\"\n weights = torch.matmul(q, k.permute(0, 1, 3, 2))\n weights /= torch.sqrt(self.key_depth_per_head)\n\n seq_len = weights.shape[-1]\n \n weights += self.position_scorer(seq_len)\n \n weights = F.softmax(weights, dim=-1)\n if self.dropout is not None:\n weights = self.dropout(weights)\n \n attn = torch.matmul(weights, v)\n\n return attn\n\n @staticmethod\n def split_last_dim(x, n):\n \"\"\"Reshape x so that the last dimension becomes two dimensions.\n The first of these two dimensions is n.\n Args:\n x (torch.Tensor): Tensor with shape (..., m)\n n (int): Size of second-to-last dimension.\n Returns:\n ret (torch.Tensor): Resulting tensor with shape (..., n, m/n)\n \"\"\"\n #import pdb;pdb.set_trace()\n old_shape = list(x.size())\n last = old_shape[-1]\n if last is not None:\n new_shape = old_shape[:-1] + [n] + [last // n]\n else:\n new_shape = old_shape[:-1] + [n]\n \n ret = x.view(new_shape)\n\n return ret.permute(0, 2, 1, 3)\n\n @staticmethod\n def combine_last_two_dim(x):\n \"\"\"Merge the last two dimensions of `x`.\n\n Args:\n x (torch.Tensor): Tensor with shape (..., m, n)\n\n Returns:\n ret (torch.Tensor): Resulting tensor with shape (..., m * n)\n \"\"\"\n old_shape = list(x.size())\n a, b = old_shape[-2:]\n new_shape = old_shape[:-2] + [a * b]\n ret = x.contiguous().view(new_shape)\n\n return ret\n" }, { "path": "models/transformer/tisa_v2.py", "content": "# Copyright 2023 Ulme Wennberg, Inc. All Rights Reserved.\n\nimport torch\nfrom torch import nn\nfrom typing import List\nimport math\n\nclass TisaV2(nn.Module):\n def __init__(self, \n num_attention_heads,\n tisa_num_kernels,\n tisa_dropout_prob,\n num_position_agnostic_heads,\n max_field_view,\n min_field_view,\n p_eps = 1e-8):\n \n super().__init__()\n\n self.num_attention_heads = num_attention_heads\n self.num_kernels = tisa_num_kernels\n self.tisa_dropout_prob = tisa_dropout_prob\n self.num_position_agnostic_heads = num_position_agnostic_heads\n self.max_field_view = max_field_view\n self.min_field_view = min_field_view\n self.p_eps = p_eps\n\n self.eps = 1e-8\n\n self.offsets = nn.Parameter(\n torch.zeros(1, self.num_kernels, self.num_attention_heads, 1, 1)\n )\n self.amplitudes = nn.Parameter(\n torch.zeros(1, self.num_kernels, self.num_attention_heads, 1, 1)\n )\n self.sharpness = nn.Parameter(\n torch.zeros(1, self.num_kernels, self.num_attention_heads, 1, 1)\n )\n\n self.bias = nn.Parameter(torch.zeros(1, self.num_attention_heads, 1, 1))\n\n self.dropout = nn.Dropout(self.tisa_dropout_prob)\n\n self.num_position_agnostic_heads = self.num_position_agnostic_heads\n self.num_position_aware_heads = (\n self.num_attention_heads - self.num_position_agnostic_heads\n )\n self.position_agnostic_heads = torch.arange(\n self.num_attention_heads - self.num_position_agnostic_heads,\n self.num_attention_heads,\n )\n\n assert 0 < self.p_eps < 1\n\n \"\"\"\n exp ( - field_view * m) = p_eps\n - log(p_eps) = field_view * m\n m = - log(p_eps) / field_view\n \"\"\"\n\n self.one_side_min_field_view = self.min_field_view / 2\n self.one_side_max_field_view = self.max_field_view / 2\n\n self.first_slope = -math.log(self.p_eps) / self.one_side_min_field_view\n self.last_slope = -math.log(self.p_eps) / self.one_side_max_field_view\n self.slopes = nn.Parameter(\n (\n self.first_slope\n * (self.last_slope / self.first_slope)\n ** (\n torch.arange(self.num_attention_heads)\n / (self.num_position_aware_heads - 1)\n )\n ).reshape(self.num_attention_heads, 1, 1),\n requires_grad=False,\n )\n\n # Disable exponential decay for position agnostic heads\n self.slopes[self.position_agnostic_heads, 0, 0] = 0.0\n\n def create_relative_offsets(self, seq_len):\n \"\"\"Creates offsets for all the relative distances between\n -seq_len + 1 to seq_len - 1.\"\"\"\n return (\n torch.arange(-seq_len, seq_len + 1, device=self.offsets.device)\n .unsqueeze(0)\n .unsqueeze(0)\n )\n\n def forward(self, dim=-1, skip_apply_dropout=False):\n \"\"\"Computes the translation-invariant positional contribution to the\n attention matrix in the self-attention module of transformer models.\"\"\"\n \n indices_from = torch.arange(dim, device=self.offsets.device).unsqueeze(0).unsqueeze(0)\n indices_to = indices_from\n\n if not self.num_kernels:\n return torch.zeros(\n (self.num_attention_heads, indices_from.shape[-1], indices_to.shape[-1])\n )\n params = (indices_to.unsqueeze(-1) - indices_from.unsqueeze(-2)).unsqueeze(-4)\n exponential_decay_arguments = -params.abs() * self.slopes\n params = params.unsqueeze(-4) - self.offsets\n params = params / self.sharpness\n params = self.amplitudes.abs() * torch.sigmoid(self.amplitudes.sign() * params)\n\n if self.training and not skip_apply_dropout:\n params = self.dropout(params)\n params = params.sum(dim=-4)\n\n # Make final dimensions completely position agnostic\n params[:, :, self.position_agnostic_heads] = 0.0\n\n params += self.eps + self.bias.abs()\n params = torch.log(params)\n params = params + exponential_decay_arguments\n return params.squeeze(0)\n\n def _init_weights(self):\n \"\"\"Initialize the weights\"\"\"\n torch.nn.init.normal_(self.offsets, mean=0.0, std=15.0)\n torch.nn.init.normal_(self.amplitudes, mean=0.0, std=0.01)\n\n self.sharpness.data.fill_(5.0)\n self.bias.data.fill_(1.0)\n" }, { "path": "pretrained_models/.gitkeep", "content": "" }, { "path": "pretrained_models/README.md", "content": "Please [download our pretrained dance models here](https://zenodo.org/record/8156769) and place them in this folder.\n" }, { "path": "pymo/Pivots.py", "content": "import numpy as np\n\nfrom pymo.Quaternions import Quaternions\n\nclass Pivots: \n \"\"\"\n Pivots is an ndarray of angular rotations\n\n This wrapper provides some functions for\n working with pivots.\n\n These are particularly useful as a number \n of atomic operations (such as adding or \n subtracting) cannot be achieved using\n the standard arithmatic and need to be\n defined differently to work correctly\n \"\"\"\n \n def __init__(self, ps): self.ps = np.array(ps)\n def __str__(self): return \"Pivots(\"+ str(self.ps) + \")\"\n def __repr__(self): return \"Pivots(\"+ repr(self.ps) + \")\"\n \n def __add__(self, other): return Pivots(np.arctan2(np.sin(self.ps + other.ps), np.cos(self.ps + other.ps)))\n def __sub__(self, other): return Pivots(np.arctan2(np.sin(self.ps - other.ps), np.cos(self.ps - other.ps)))\n def __mul__(self, other): return Pivots(self.ps * other.ps)\n def __div__(self, other): return Pivots(self.ps / other.ps)\n def __mod__(self, other): return Pivots(self.ps % other.ps)\n def __pow__(self, other): return Pivots(self.ps ** other.ps)\n \n def __lt__(self, other): return self.ps < other.ps\n def __le__(self, other): return self.ps <= other.ps\n def __eq__(self, other): return self.ps == other.ps\n def __ne__(self, other): return self.ps != other.ps\n def __ge__(self, other): return self.ps >= other.ps\n def __gt__(self, other): return self.ps > other.ps\n \n def __abs__(self): return Pivots(abs(self.ps))\n def __neg__(self): return Pivots(-self.ps)\n \n def __iter__(self): return iter(self.ps)\n def __len__(self): return len(self.ps)\n \n def __getitem__(self, k): return Pivots(self.ps[k]) \n def __setitem__(self, k, v): self.ps[k] = v.ps\n \n def _ellipsis(self): return tuple(map(lambda x: slice(None), self.shape))\n \n def quaternions(self, plane='xz'):\n fa = self._ellipsis()\n axises = np.ones(self.ps.shape + (3,))\n axises[fa + (\"xyz\".index(plane[0]),)] = 0.0\n axises[fa + (\"xyz\".index(plane[1]),)] = 0.0\n return Quaternions.from_angle_axis(self.ps, axises)\n \n def directions(self, plane='xz'):\n dirs = np.zeros((len(self.ps), 3))\n dirs[\"xyz\".index(plane[0])] = np.sin(self.ps)\n dirs[\"xyz\".index(plane[1])] = np.cos(self.ps)\n return dirs\n \n def normalized(self):\n xs = np.copy(self.ps)\n while np.any(xs > np.pi): xs[xs > np.pi] = xs[xs > np.pi] - 2 * np.pi\n while np.any(xs < -np.pi): xs[xs < -np.pi] = xs[xs < -np.pi] + 2 * np.pi\n return Pivots(xs)\n \n def interpolate(self, ws):\n dir = np.average(self.directions, weights=ws, axis=0)\n return np.arctan2(dir[2], dir[0])\n \n def copy(self):\n return Pivots(np.copy(self.ps))\n \n @property\n def shape(self):\n return self.ps.shape\n \n @classmethod\n def from_quaternions(cls, qs, forward='z', plane='xz'):\n ds = np.zeros(qs.shape + (3,))\n ds[...,'xyz'.index(forward)] = 1.0\n return Pivots.from_directions(qs * ds, plane=plane)\n \n @classmethod\n def from_directions(cls, ds, plane='xz'):\n ys = ds[...,'xyz'.index(plane[0])]\n xs = ds[...,'xyz'.index(plane[1])]\n return Pivots(np.arctan2(ys, xs))\n \n" }, { "path": "pymo/Quaternions.py", "content": "import numpy as np\n\nclass Quaternions:\n \"\"\"\n Quaternions is a wrapper around a numpy ndarray\n that allows it to act as if it were an narray of\n a quaternion data type.\n \n Therefore addition, subtraction, multiplication,\n division, negation, absolute, are all defined\n in terms of quaternion operations such as quaternion\n multiplication.\n \n This allows for much neater code and many routines\n which conceptually do the same thing to be written\n in the same way for point data and for rotation data.\n \n The Quaternions class has been desgined such that it\n should support broadcasting and slicing in all of the\n usual ways.\n \"\"\"\n \n def __init__(self, qs):\n if isinstance(qs, np.ndarray):\n \n if len(qs.shape) == 1: qs = np.array([qs])\n self.qs = qs\n return\n \n if isinstance(qs, Quaternions):\n self.qs = qs.qs\n return\n \n raise TypeError('Quaternions must be constructed from iterable, numpy array, or Quaternions, not %s' % type(qs))\n \n def __str__(self): return \"Quaternions(\"+ str(self.qs) + \")\"\n def __repr__(self): return \"Quaternions(\"+ repr(self.qs) + \")\"\n \n \"\"\" Helper Methods for Broadcasting and Data extraction \"\"\"\n \n @classmethod\n def _broadcast(cls, sqs, oqs, scalar=False):\n \n if isinstance(oqs, float): return sqs, oqs * np.ones(sqs.shape[:-1])\n \n ss = np.array(sqs.shape) if not scalar else np.array(sqs.shape[:-1])\n os = np.array(oqs.shape)\n \n if len(ss) != len(os):\n raise TypeError('Quaternions cannot broadcast together shapes %s and %s' % (sqs.shape, oqs.shape))\n \n if np.all(ss == os): return sqs, oqs\n \n if not np.all((ss == os) | (os == np.ones(len(os))) | (ss == np.ones(len(ss)))):\n raise TypeError('Quaternions cannot broadcast together shapes %s and %s' % (sqs.shape, oqs.shape))\n \n sqsn, oqsn = sqs.copy(), oqs.copy()\n \n for a in np.where(ss == 1)[0]: sqsn = sqsn.repeat(os[a], axis=a)\n for a in np.where(os == 1)[0]: oqsn = oqsn.repeat(ss[a], axis=a)\n \n return sqsn, oqsn\n \n \"\"\" Adding Quaterions is just Defined as Multiplication \"\"\"\n \n def __add__(self, other): return self * other\n def __sub__(self, other): return self / other\n \n \"\"\" Quaterion Multiplication \"\"\"\n \n def __mul__(self, other):\n \"\"\"\n Quaternion multiplication has three main methods.\n \n When multiplying a Quaternions array by Quaternions\n normal quaternion multiplication is performed.\n \n When multiplying a Quaternions array by a vector\n array of the same shape, where the last axis is 3,\n it is assumed to be a Quaternion by 3D-Vector \n multiplication and the 3D-Vectors are rotated\n in space by the Quaternions.\n \n When multipplying a Quaternions array by a scalar\n or vector of different shape it is assumed to be\n a Quaternions by Scalars multiplication and the\n Quaternions are scaled using Slerp and the identity\n quaternions.\n \"\"\"\n \n \"\"\" If Quaternions type do Quaternions * Quaternions \"\"\"\n if isinstance(other, Quaternions):\n \n sqs, oqs = Quaternions._broadcast(self.qs, other.qs)\n \n q0 = sqs[...,0]; q1 = sqs[...,1]; \n q2 = sqs[...,2]; q3 = sqs[...,3]; \n r0 = oqs[...,0]; r1 = oqs[...,1]; \n r2 = oqs[...,2]; r3 = oqs[...,3]; \n \n qs = np.empty(sqs.shape)\n qs[...,0] = r0 * q0 - r1 * q1 - r2 * q2 - r3 * q3\n qs[...,1] = r0 * q1 + r1 * q0 - r2 * q3 + r3 * q2\n qs[...,2] = r0 * q2 + r1 * q3 + r2 * q0 - r3 * q1\n qs[...,3] = r0 * q3 - r1 * q2 + r2 * q1 + r3 * q0\n \n return Quaternions(qs)\n \n \"\"\" If array type do Quaternions * Vectors \"\"\"\n if isinstance(other, np.ndarray) and other.shape[-1] == 3:\n vs = Quaternions(np.concatenate([np.zeros(other.shape[:-1] + (1,)), other], axis=-1))\n return (self * (vs * -self)).imaginaries\n \n \"\"\" If float do Quaternions * Scalars \"\"\"\n if isinstance(other, np.ndarray) or isinstance(other, float):\n return Quaternions.slerp(Quaternions.id_like(self), self, other)\n \n raise TypeError('Cannot multiply/add Quaternions with type %s' % str(type(other)))\n \n def __div__(self, other):\n \"\"\"\n When a Quaternion type is supplied, division is defined\n as multiplication by the inverse of that Quaternion.\n \n When a scalar or vector is supplied it is defined\n as multiplicaion of one over the supplied value.\n Essentially a scaling.\n \"\"\"\n \n if isinstance(other, Quaternions): return self * (-other)\n if isinstance(other, np.ndarray): return self * (1.0 / other)\n if isinstance(other, float): return self * (1.0 / other)\n raise TypeError('Cannot divide/subtract Quaternions with type %s' + str(type(other)))\n \n def __eq__(self, other): return self.qs == other.qs\n def __ne__(self, other): return self.qs != other.qs\n \n def __neg__(self):\n \"\"\" Invert Quaternions \"\"\"\n return Quaternions(self.qs * np.array([[1, -1, -1, -1]]))\n \n def __abs__(self):\n \"\"\" Unify Quaternions To Single Pole \"\"\"\n qabs = self.normalized().copy()\n top = np.sum(( qabs.qs) * np.array([1,0,0,0]), axis=-1)\n bot = np.sum((-qabs.qs) * np.array([1,0,0,0]), axis=-1)\n qabs.qs[top < bot] = -qabs.qs[top < bot]\n return qabs\n \n def __iter__(self): return iter(self.qs)\n def __len__(self): return len(self.qs)\n \n def __getitem__(self, k): return Quaternions(self.qs[k]) \n def __setitem__(self, k, v): self.qs[k] = v.qs\n \n @property\n def lengths(self):\n return np.sum(self.qs**2.0, axis=-1)**0.5\n \n @property\n def reals(self):\n return self.qs[...,0]\n \n @property\n def imaginaries(self):\n return self.qs[...,1:4]\n \n @property\n def shape(self): return self.qs.shape[:-1]\n \n def repeat(self, n, **kwargs):\n return Quaternions(self.qs.repeat(n, **kwargs))\n \n def normalized(self):\n return Quaternions(self.qs / self.lengths[...,np.newaxis])\n \n def log(self):\n norm = abs(self.normalized())\n imgs = norm.imaginaries\n lens = np.sqrt(np.sum(imgs**2, axis=-1))\n lens = np.arctan2(lens, norm.reals) / (lens + 1e-10)\n return imgs * lens[...,np.newaxis]\n \n def constrained(self, axis):\n \n rl = self.reals\n im = np.sum(axis * self.imaginaries, axis=-1)\n \n t1 = -2 * np.arctan2(rl, im) + np.pi\n t2 = -2 * np.arctan2(rl, im) - np.pi\n \n top = Quaternions.exp(axis[np.newaxis] * (t1[:,np.newaxis] / 2.0))\n bot = Quaternions.exp(axis[np.newaxis] * (t2[:,np.newaxis] / 2.0))\n img = self.dot(top) > self.dot(bot)\n \n ret = top.copy()\n ret[ img] = top[ img]\n ret[~img] = bot[~img]\n return ret\n \n def constrained_x(self): return self.constrained(np.array([1,0,0]))\n def constrained_y(self): return self.constrained(np.array([0,1,0]))\n def constrained_z(self): return self.constrained(np.array([0,0,1]))\n \n def dot(self, q): return np.sum(self.qs * q.qs, axis=-1)\n \n def copy(self): return Quaternions(np.copy(self.qs))\n \n def reshape(self, s):\n self.qs.reshape(s)\n return self\n \n def interpolate(self, ws):\n return Quaternions.exp(np.average(abs(self).log, axis=0, weights=ws))\n \n def euler(self, order='xyz'):\n \n q = self.normalized().qs\n q0 = q[...,0]\n q1 = q[...,1]\n q2 = q[...,2]\n q3 = q[...,3]\n es = np.zeros(self.shape + (3,))\n \n if order == 'xyz':\n es[...,0] = np.arctan2(2 * (q0 * q1 + q2 * q3), 1 - 2 * (q1 * q1 + q2 * q2))\n es[...,1] = np.arcsin((2 * (q0 * q2 - q3 * q1)).clip(-1,1))\n es[...,2] = np.arctan2(2 * (q0 * q3 + q1 * q2), 1 - 2 * (q2 * q2 + q3 * q3))\n elif order == 'yzx':\n es[...,0] = np.arctan2(2 * (q1 * q0 - q2 * q3), -q1 * q1 + q2 * q2 - q3 * q3 + q0 * q0)\n es[...,1] = np.arctan2(2 * (q2 * q0 - q1 * q3), q1 * q1 - q2 * q2 - q3 * q3 + q0 * q0)\n es[...,2] = np.arcsin((2 * (q1 * q2 + q3 * q0)).clip(-1,1))\n else:\n raise NotImplementedError('Cannot convert from ordering %s' % order)\n \n \"\"\"\n \n # These conversion don't appear to work correctly for Maya.\n # http://bediyap.com/programming/convert-quaternion-to-euler-rotations/\n \n if order == 'xyz':\n es[...,0] = np.arctan2(2 * (q0 * q3 - q1 * q2), q0 * q0 + q1 * q1 - q2 * q2 - q3 * q3)\n es[...,1] = np.arcsin((2 * (q1 * q3 + q0 * q2)).clip(-1,1))\n es[...,2] = np.arctan2(2 * (q0 * q1 - q2 * q3), q0 * q0 - q1 * q1 - q2 * q2 + q3 * q3)\n elif order == 'yzx':\n es[...,0] = np.arctan2(2 * (q0 * q1 - q2 * q3), q0 * q0 - q1 * q1 + q2 * q2 - q3 * q3)\n es[...,1] = np.arcsin((2 * (q1 * q2 + q0 * q3)).clip(-1,1))\n es[...,2] = np.arctan2(2 * (q0 * q2 - q1 * q3), q0 * q0 + q1 * q1 - q2 * q2 - q3 * q3)\n elif order == 'zxy':\n es[...,0] = np.arctan2(2 * (q0 * q2 - q1 * q3), q0 * q0 - q1 * q1 - q2 * q2 + q3 * q3)\n es[...,1] = np.arcsin((2 * (q0 * q1 + q2 * q3)).clip(-1,1))\n es[...,2] = np.arctan2(2 * (q0 * q3 - q1 * q2), q0 * q0 - q1 * q1 + q2 * q2 - q3 * q3) \n elif order == 'xzy':\n es[...,0] = np.arctan2(2 * (q0 * q2 + q1 * q3), q0 * q0 + q1 * q1 - q2 * q2 - q3 * q3)\n es[...,1] = np.arcsin((2 * (q0 * q3 - q1 * q2)).clip(-1,1))\n es[...,2] = np.arctan2(2 * (q0 * q1 + q2 * q3), q0 * q0 - q1 * q1 + q2 * q2 - q3 * q3)\n elif order == 'yxz':\n es[...,0] = np.arctan2(2 * (q1 * q2 + q0 * q3), q0 * q0 - q1 * q1 + q2 * q2 - q3 * q3)\n es[...,1] = np.arcsin((2 * (q0 * q1 - q2 * q3)).clip(-1,1))\n es[...,2] = np.arctan2(2 * (q1 * q3 + q0 * q2), q0 * q0 - q1 * q1 - q2 * q2 + q3 * q3)\n elif order == 'zyx':\n es[...,0] = np.arctan2(2 * (q0 * q1 + q2 * q3), q0 * q0 - q1 * q1 - q2 * q2 + q3 * q3)\n es[...,1] = np.arcsin((2 * (q0 * q2 - q1 * q3)).clip(-1,1))\n es[...,2] = np.arctan2(2 * (q0 * q3 + q1 * q2), q0 * q0 + q1 * q1 - q2 * q2 - q3 * q3)\n else:\n raise KeyError('Unknown ordering %s' % order)\n \n \"\"\"\n \n # https://github.com/ehsan/ogre/blob/master/OgreMain/src/OgreMatrix3.cpp\n # Use this class and convert from matrix\n \n return es\n \n \n def average(self):\n \n if len(self.shape) == 1:\n \n import numpy.core.umath_tests as ut\n system = ut.matrix_multiply(self.qs[:,:,np.newaxis], self.qs[:,np.newaxis,:]).sum(axis=0)\n w, v = np.linalg.eigh(system)\n qiT_dot_qref = (self.qs[:,:,np.newaxis] * v[np.newaxis,:,:]).sum(axis=1)\n return Quaternions(v[:,np.argmin((1.-qiT_dot_qref**2).sum(axis=0))]) \n \n else:\n \n raise NotImplementedError('Cannot average multi-dimensionsal Quaternions')\n\n def angle_axis(self):\n \n norm = self.normalized() \n s = np.sqrt(1 - (norm.reals**2.0))\n s[s == 0] = 0.001\n \n angles = 2.0 * np.arccos(norm.reals)\n axis = norm.imaginaries / s[...,np.newaxis]\n \n return angles, axis\n \n \n def transforms(self):\n \n qw = self.qs[...,0]\n qx = self.qs[...,1]\n qy = self.qs[...,2]\n qz = self.qs[...,3]\n \n x2 = qx + qx; y2 = qy + qy; z2 = qz + qz;\n xx = qx * x2; yy = qy * y2; wx = qw * x2;\n xy = qx * y2; yz = qy * z2; wy = qw * y2;\n xz = qx * z2; zz = qz * z2; wz = qw * z2;\n \n m = np.empty(self.shape + (3,3))\n m[...,0,0] = 1.0 - (yy + zz)\n m[...,0,1] = xy - wz\n m[...,0,2] = xz + wy \n m[...,1,0] = xy + wz\n m[...,1,1] = 1.0 - (xx + zz)\n m[...,1,2] = yz - wx \n m[...,2,0] = xz - wy\n m[...,2,1] = yz + wx\n m[...,2,2] = 1.0 - (xx + yy)\n \n return m\n \n def ravel(self):\n return self.qs.ravel()\n \n @classmethod\n def id(cls, n):\n \n if isinstance(n, tuple):\n qs = np.zeros(n + (4,))\n qs[...,0] = 1.0\n return Quaternions(qs)\n \n if isinstance(n, int) or isinstance(n, long):\n qs = np.zeros((n,4))\n qs[:,0] = 1.0\n return Quaternions(qs)\n \n raise TypeError('Cannot Construct Quaternion from %s type' % str(type(n)))\n\n @classmethod\n def id_like(cls, a):\n qs = np.zeros(a.shape + (4,))\n qs[...,0] = 1.0\n return Quaternions(qs)\n \n @classmethod\n def exp(cls, ws):\n \n ts = np.sum(ws**2.0, axis=-1)**0.5\n ts[ts == 0] = 0.001\n ls = np.sin(ts) / ts\n \n qs = np.empty(ws.shape[:-1] + (4,))\n qs[...,0] = np.cos(ts)\n qs[...,1] = ws[...,0] * ls\n qs[...,2] = ws[...,1] * ls\n qs[...,3] = ws[...,2] * ls\n \n return Quaternions(qs).normalized()\n \n @classmethod\n def slerp(cls, q0s, q1s, a):\n \n fst, snd = cls._broadcast(q0s.qs, q1s.qs)\n fst, a = cls._broadcast(fst, a, scalar=True)\n snd, a = cls._broadcast(snd, a, scalar=True)\n \n len = np.sum(fst * snd, axis=-1)\n \n neg = len < 0.0\n len[neg] = -len[neg]\n snd[neg] = -snd[neg]\n \n amount0 = np.zeros(a.shape)\n amount1 = np.zeros(a.shape)\n\n linear = (1.0 - len) < 0.01\n omegas = np.arccos(len[~linear])\n sinoms = np.sin(omegas)\n \n amount0[ linear] = 1.0 - a[linear]\n amount1[ linear] = a[linear]\n amount0[~linear] = np.sin((1.0 - a[~linear]) * omegas) / sinoms\n amount1[~linear] = np.sin( a[~linear] * omegas) / sinoms\n \n return Quaternions(\n amount0[...,np.newaxis] * fst + \n amount1[...,np.newaxis] * snd)\n \n @classmethod\n def between(cls, v0s, v1s):\n a = np.cross(v0s, v1s)\n w = np.sqrt((v0s**2).sum(axis=-1) * (v1s**2).sum(axis=-1)) + (v0s * v1s).sum(axis=-1)\n return Quaternions(np.concatenate([w[...,np.newaxis], a], axis=-1)).normalized()\n \n @classmethod\n def from_angle_axis(cls, angles, axis):\n axis = axis / (np.sqrt(np.sum(axis**2, axis=-1)) + 1e-10)[...,np.newaxis]\n sines = np.sin(angles / 2.0)[...,np.newaxis]\n cosines = np.cos(angles / 2.0)[...,np.newaxis]\n return Quaternions(np.concatenate([cosines, axis * sines], axis=-1))\n \n @classmethod\n def from_euler(cls, es, order='xyz', world=False):\n \n axis = {\n 'x' : np.array([1,0,0]),\n 'y' : np.array([0,1,0]),\n 'z' : np.array([0,0,1]),\n }\n \n q0s = Quaternions.from_angle_axis(es[...,0], axis[order[0]])\n q1s = Quaternions.from_angle_axis(es[...,1], axis[order[1]])\n q2s = Quaternions.from_angle_axis(es[...,2], axis[order[2]])\n \n return (q2s * (q1s * q0s)) if world else (q0s * (q1s * q2s))\n \n @classmethod\n def from_transforms(cls, ts):\n \n d0, d1, d2 = ts[...,0,0], ts[...,1,1], ts[...,2,2]\n \n q0 = ( d0 + d1 + d2 + 1.0) / 4.0\n q1 = ( d0 - d1 - d2 + 1.0) / 4.0\n q2 = (-d0 + d1 - d2 + 1.0) / 4.0\n q3 = (-d0 - d1 + d2 + 1.0) / 4.0\n \n q0 = np.sqrt(q0.clip(0,None))\n q1 = np.sqrt(q1.clip(0,None))\n q2 = np.sqrt(q2.clip(0,None))\n q3 = np.sqrt(q3.clip(0,None))\n \n c0 = (q0 >= q1) & (q0 >= q2) & (q0 >= q3)\n c1 = (q1 >= q0) & (q1 >= q2) & (q1 >= q3)\n c2 = (q2 >= q0) & (q2 >= q1) & (q2 >= q3)\n c3 = (q3 >= q0) & (q3 >= q1) & (q3 >= q2)\n \n q1[c0] *= np.sign(ts[c0,2,1] - ts[c0,1,2])\n q2[c0] *= np.sign(ts[c0,0,2] - ts[c0,2,0])\n q3[c0] *= np.sign(ts[c0,1,0] - ts[c0,0,1])\n \n q0[c1] *= np.sign(ts[c1,2,1] - ts[c1,1,2])\n q2[c1] *= np.sign(ts[c1,1,0] + ts[c1,0,1])\n q3[c1] *= np.sign(ts[c1,0,2] + ts[c1,2,0]) \n \n q0[c2] *= np.sign(ts[c2,0,2] - ts[c2,2,0])\n q1[c2] *= np.sign(ts[c2,1,0] + ts[c2,0,1])\n q3[c2] *= np.sign(ts[c2,2,1] + ts[c2,1,2]) \n \n q0[c3] *= np.sign(ts[c3,1,0] - ts[c3,0,1])\n q1[c3] *= np.sign(ts[c3,2,0] + ts[c3,0,2])\n q2[c3] *= np.sign(ts[c3,2,1] + ts[c3,1,2]) \n \n qs = np.empty(ts.shape[:-2] + (4,))\n qs[...,0] = q0\n qs[...,1] = q1\n qs[...,2] = q2\n qs[...,3] = q3\n \n return cls(qs)\n \n \n " }, { "path": "pymo/__init__.py", "content": "" }, { "path": "pymo/data.py", "content": "import numpy as np\n\nclass Joint():\n def __init__(self, name, parent=None, children=None):\n self.name = name\n self.parent = parent\n self.children = children\n\nclass MocapData():\n def __init__(self):\n self.skeleton = {}\n self.values = None\n self.channel_names = []\n self.framerate = 0.0\n self.root_name = ''\n self.take_name = ''\n \n def traverse(self, j=None):\n stack = [self.root_name]\n while stack:\n joint = stack.pop()\n yield joint\n for c in self.skeleton[joint]['children']:\n stack.append(c)\n\n def clone(self):\n import copy\n new_data = MocapData()\n new_data.skeleton = copy.deepcopy(self.skeleton)\n new_data.values = copy.deepcopy(self.values)\n new_data.channel_names = copy.deepcopy(self.channel_names)\n new_data.root_name = copy.deepcopy(self.root_name)\n new_data.framerate = copy.deepcopy(self.framerate)\n if hasattr(self,'take_name'):\n new_data.take_name = copy.deepcopy(self.take_name)\n return new_data\n\n def get_all_channels(self):\n '''Returns all of the channels parsed from the file as a 2D numpy array'''\n\n frames = [f[1] for f in self.values]\n return np.asarray([[channel[2] for channel in frame] for frame in frames])\n\n def get_skeleton_tree(self):\n tree = []\n root_key = [j for j in self.skeleton if self.skeleton[j]['parent']==None][0]\n \n root_joint = Joint(root_key)\n \n def get_empty_channels(self):\n #TODO\n pass\n\n def get_constant_channels(self):\n #TODO\n pass\n" }, { "path": "pymo/features.py", "content": "'''\nA set of mocap feature extraction functions\n\nCreated by Omid Alemi | Nov 17 2017\n\n'''\nimport numpy as np\nimport pandas as pd\nimport peakutils\nimport matplotlib.pyplot as plt\n\ndef get_foot_contact_idxs(signal, t=0.02, min_dist=120):\n up_idxs = peakutils.indexes(signal, thres=t/max(signal), min_dist=min_dist)\n down_idxs = peakutils.indexes(-signal, thres=t/min(signal), min_dist=min_dist)\n\n return [up_idxs, down_idxs]\n\n\ndef create_foot_contact_signal(mocap_track, col_name, start=1, t=0.02, min_dist=120):\n signal = mocap_track.values[col_name].values\n idxs = get_foot_contact_idxs(signal, t, min_dist)\n\n step_signal = []\n\n c = start\n for f in range(len(signal)): \n if f in idxs[1]:\n c = 0\n elif f in idxs[0]:\n c = 1\n \n step_signal.append(c)\n \n return step_signal\n\ndef plot_foot_up_down(mocap_track, col_name, t=0.02, min_dist=120):\n \n signal = mocap_track.values[col_name].values\n idxs = get_foot_contact_idxs(signal, t, min_dist)\n\n plt.plot(mocap_track.values.index, signal)\n plt.plot(mocap_track.values.index[idxs[0]], signal[idxs[0]], 'ro')\n plt.plot(mocap_track.values.index[idxs[1]], signal[idxs[1]], 'go')\n" }, { "path": "pymo/parsers.py", "content": "'''\nBVH Parser Class\n\nBy Omid Alemi\nCreated: June 12, 2017\n\nBased on: https://gist.github.com/johnfredcee/2007503\n\n'''\nimport os\nimport re\nimport numpy as np\nfrom pymo.data import Joint, MocapData\n\nclass BVHScanner():\n '''\n A wrapper class for re.Scanner\n '''\n def __init__(self):\n\n def identifier(scanner, token):\n return 'IDENT', token\n\n def operator(scanner, token):\n return 'OPERATOR', token\n\n def digit(scanner, token):\n return 'DIGIT', token\n\n def open_brace(scanner, token):\n return 'OPEN_BRACE', token\n\n def close_brace(scanner, token):\n return 'CLOSE_BRACE', token\n\n self.scanner = re.Scanner([\n (r'[a-zA-Z_]\\w*', identifier),\n #(r'-*[0-9]+(\\.[0-9]+)?', digit), # won't work for .34\n #(r'[-+]?[0-9]*\\.?[0-9]+', digit), # won't work for 4.56e-2\n #(r'[-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?', digit),\n (r'-*[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?', digit),\n (r'}', close_brace),\n (r'}', close_brace),\n (r'{', open_brace),\n (r':', None),\n (r'\\s+', None)\n ])\n\n def scan(self, stuff):\n return self.scanner.scan(stuff)\n\n\n\nclass BVHParser():\n '''\n A class to parse a BVH file.\n \n Extracts the skeleton and channel values\n '''\n def __init__(self, filename=None):\n self.reset()\n\n def reset(self): \n self._skeleton = {}\n self.bone_context = []\n self._motion_channels = []\n self._motions = []\n self.current_token = 0\n self.framerate = 0.0\n self.root_name = ''\n\n self.scanner = BVHScanner()\n \n self.data = MocapData()\n\n\n def parse(self, filename, start=0, stop=-1):\n self.reset()\n\n with open(filename, 'r') as bvh_file:\n raw_contents = bvh_file.read()\n tokens, remainder = self.scanner.scan(raw_contents)\n self._parse_hierarchy(tokens)\n self.current_token = self.current_token + 1\n self._parse_motion(tokens, start, stop)\n self.data.skeleton = self._skeleton\n self.data.channel_names = self._motion_channels\n self.data.values = self._to_DataFrame()\n self.data.root_name = self.root_name\n self.data.framerate = self.framerate\n self.data.take_name = os.path.basename(os.path.splitext(filename)[0])\n\n return self.data\n \n def _to_DataFrame(self):\n '''Returns all of the channels parsed from the file as a pandas DataFrame'''\n\n import pandas as pd\n time_index = pd.to_timedelta([f[0] for f in self._motions], unit='s')\n frames = [f[1] for f in self._motions]\n channels = np.asarray([[channel[2] for channel in frame] for frame in frames])\n column_names = ['%s_%s'%(c[0], c[1]) for c in self._motion_channels]\n\n return pd.DataFrame(data=channels, index=time_index, columns=column_names)\n\n\n def _new_bone(self, parent, name):\n bone = {'parent': parent, 'channels': [], 'offsets': [], 'order': '','children': []}\n return bone\n\n def _push_bone_context(self,name):\n self.bone_context.append(name)\n\n def _get_bone_context(self):\n return self.bone_context[len(self.bone_context)-1]\n\n def _pop_bone_context(self):\n self.bone_context = self.bone_context[:-1]\n return self.bone_context[len(self.bone_context)-1]\n\n def _read_offset(self, bvh, token_index):\n if bvh[token_index] != ('IDENT', 'OFFSET'):\n return None, None\n token_index = token_index + 1\n offsets = [0.0] * 3\n for i in range(3):\n offsets[i] = float(bvh[token_index][1])\n token_index = token_index + 1\n return offsets, token_index\n \n def _read_channels(self, bvh, token_index):\n if bvh[token_index] != ('IDENT', 'CHANNELS'):\n return None, None\n token_index = token_index + 1\n channel_count = int(bvh[token_index][1])\n token_index = token_index + 1\n channels = [\"\"] * channel_count\n order = \"\"\n for i in range(channel_count):\n channels[i] = bvh[token_index][1]\n token_index = token_index + 1\n if(channels[i] == \"Xrotation\" or channels[i]== \"Yrotation\" or channels[i]== \"Zrotation\"):\n order += channels[i][0]\n else :\n order = \"\"\n return channels, token_index, order\n\n def _parse_joint(self, bvh, token_index):\n end_site = False\n joint_id = bvh[token_index][1]\n token_index = token_index + 1\n joint_name = bvh[token_index][1]\n token_index = token_index + 1\n \n parent_name = self._get_bone_context()\n\n if (joint_id == \"End\"):\n joint_name = parent_name+ '_Nub'\n end_site = True\n joint = self._new_bone(parent_name, joint_name)\n if bvh[token_index][0] != 'OPEN_BRACE':\n print('Was expecting brance, got ', bvh[token_index])\n return None\n token_index = token_index + 1\n offsets, token_index = self._read_offset(bvh, token_index)\n joint['offsets'] = offsets\n if not end_site:\n channels, token_index, order = self._read_channels(bvh, token_index)\n joint['channels'] = channels\n joint['order'] = order\n for channel in channels:\n self._motion_channels.append((joint_name, channel))\n\n self._skeleton[joint_name] = joint\n self._skeleton[parent_name]['children'].append(joint_name)\n\n while (bvh[token_index][0] == 'IDENT' and bvh[token_index][1] == 'JOINT') or (bvh[token_index][0] == 'IDENT' and bvh[token_index][1] == 'End'):\n self._push_bone_context(joint_name)\n token_index = self._parse_joint(bvh, token_index)\n self._pop_bone_context()\n\n if bvh[token_index][0] == 'CLOSE_BRACE':\n return token_index + 1\n\n print('Unexpected token ', bvh[token_index])\n\n def _parse_hierarchy(self, bvh):\n self.current_token = 0\n if bvh[self.current_token] != ('IDENT', 'HIERARCHY'):\n return None\n self.current_token = self.current_token + 1\n if bvh[self.current_token] != ('IDENT', 'ROOT'):\n return None\n self.current_token = self.current_token + 1\n if bvh[self.current_token][0] != 'IDENT':\n return None\n\n root_name = bvh[self.current_token][1]\n root_bone = self._new_bone(None, root_name)\n self.current_token = self.current_token + 2 #skipping open brace\n offsets, self.current_token = self._read_offset(bvh, self.current_token)\n channels, self.current_token, order = self._read_channels(bvh, self.current_token)\n root_bone['offsets'] = offsets\n root_bone['channels'] = channels\n root_bone['order'] = order\n self._skeleton[root_name] = root_bone\n self._push_bone_context(root_name)\n \n for channel in channels:\n self._motion_channels.append((root_name, channel))\n\n while bvh[self.current_token][1] == 'JOINT' or bvh[self.current_token][1] == 'End':\n self.current_token = self._parse_joint(bvh, self.current_token)\n \n self.root_name = root_name\n\n def _parse_motion(self, bvh, start, stop):\n if bvh[self.current_token][0] != 'IDENT':\n print('Unexpected text')\n return None\n if bvh[self.current_token][1] != 'MOTION':\n print('No motion section')\n return None\n self.current_token = self.current_token + 1\n if bvh[self.current_token][1] != 'Frames':\n return None\n self.current_token = self.current_token + 1\n frame_count = int(bvh[self.current_token][1])\n \n if stop<0 or stop>frame_count:\n stop = frame_count\n \n assert(start>=0)\n assert(start=start:\n self._motions[idx] = (frame_time, channel_values)\n frame_time = frame_time + frame_rate\n idx+=1\n" }, { "path": "pymo/preprocessing.py", "content": "'''\nPreprocessing Tranformers Based on sci-kit's API\n\nBy Omid Alemi\nCreated on June 12, 2017\n'''\nimport copy\nimport pandas as pd\nimport numpy as np\nimport transforms3d as t3d\nimport scipy.ndimage.filters as filters\nfrom scipy.spatial.transform import Rotation as R\n\nfrom scipy import signal, interpolate\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.pipeline import Pipeline\n\nfrom pymo.rotation_tools import Rotation, euler2expmap, euler2expmap2, expmap2euler, euler_reorder, unroll, euler2vectors, vectors2euler\nfrom pymo.Quaternions import Quaternions\nfrom pymo.Pivots import Pivots\n\nclass MocapParameterizer(BaseEstimator, TransformerMixin):\n def __init__(self, param_type = 'euler', ref_pose=None):\n '''\n \n param_type = {'euler', 'quat', 'expmap', 'position', 'expmap2pos'}\n '''\n self.param_type = param_type\n if (ref_pose is not None):\n self.ref_pose = self._to_quat(ref_pose)[0]\n else:\n self.ref_pose = None\n\n def fit(self, X, y=None):\n return self\n\n def transform(self, X, y=None):\n #print(\"MocapParameterizer: \" + self.param_type)\n if self.param_type == 'euler':\n return X\n elif self.param_type == 'expmap':\n if self.ref_pose is None:\n return self._to_expmap(X)\n else:\n return self._to_expmap2(X)\n elif self.param_type == 'vectors':\n return self._euler_to_vectors(X)\n elif self.param_type == 'quat':\n return self._to_quat(X)\n elif self.param_type == 'position':\n return self._to_pos(X)\n elif self.param_type == 'expmap2pos':\n return self._expmap_to_pos(X)\n else:\n raise 'param types: euler, quat, expmap, position, expmap2pos'\n\n# return X\n \n def inverse_transform(self, X, copy=None): \n if self.param_type == 'euler':\n return X\n elif self.param_type == 'expmap':\n if self.ref_pose is None:\n return self._expmap_to_euler(X)\n else:\n return self._expmap_to_euler2(X) \n elif self.param_type == 'vectors':\n return self._vectors_to_euler(X)\n elif self.param_type == 'quat':\n return self._quat_to_euler(X)\n elif self.param_type == 'position':\n # raise 'positions 2 eulers is not supported'\n print('positions 2 eulers is not supported')\n return X\n else:\n raise 'param types: euler, quat, expmap, position'\n\n def _to_quat(self, X):\n '''Converts joints rotations in quaternions'''\n\n Q = []\n for track in X:\n channels = []\n titles = []\n euler_df = track.values\n\n # Create a new DataFrame to store the exponential map rep\n quat_df = euler_df.copy()\n\n # List the columns that contain rotation channels\n rot_cols = [c for c in euler_df.columns if ('rotation' in c and 'Nub' not in c)]\n\n # List the joints that are not end sites, i.e., have channels\n joints = (joint for joint in track.skeleton if 'Nub' not in joint)\n\n for joint in joints:\n rot_order = track.skeleton[joint]['order']\n\n # Get the rotation columns that belong to this joint\n rc = euler_df[[c for c in rot_cols if joint in c]]\n\n r1_col = '%s_%srotation'%(joint, rot_order[0])\n r2_col = '%s_%srotation'%(joint, rot_order[1])\n r3_col = '%s_%srotation'%(joint, rot_order[2])\n # Make sure the columns are organized in xyz order\n if rc.shape[1] < 3:\n euler_values = np.zeros((euler_df.shape[0], 3))\n rot_order = \"XYZ\"\n else:\n euler_values = np.pi/180.0*np.transpose(np.array([track.values[r1_col], track.values[r2_col], track.values[r3_col]]))\n \n quat_df.drop([r1_col, r2_col, r3_col], axis=1, inplace=True)\n quats = Quaternions.from_euler(np.asarray(euler_values), order=rot_order.lower(), world=False)\n\n # Create the corresponding columns in the new DataFrame\n quat_df['%s_qWrotation'%joint] = pd.Series(data=[e[0] for e in quats], index=quat_df.index)\n quat_df['%s_qXrotation'%joint] = pd.Series(data=[e[1] for e in quats], index=quat_df.index)\n quat_df['%s_qYrotation'%joint] = pd.Series(data=[e[2] for e in quats], index=quat_df.index)\n quat_df['%s_qZrotation'%joint] = pd.Series(data=[e[3] for e in quats], index=quat_df.index)\n\n new_track = track.clone()\n new_track.values = quat_df\n Q.append(new_track)\n return Q\n \n def _quat_to_euler(self, X):\n Q = []\n for track in X:\n channels = []\n titles = []\n quat_df = track.values\n\n # Create a new DataFrame to store the exponential map rep\n #euler_df = pd.DataFrame(index=exp_df.index)\n euler_df = quat_df.copy()\n\n # List the columns that contain rotation channels\n quat_params = [c for c in quat_df.columns if ( any(p in c for p in ['qWrotation','qXrotation','qYrotation','qZrotation']) and 'Nub' not in c)]\n\n # List the joints that are not end sites, i.e., have channels\n joints = (joint for joint in track.skeleton if 'Nub' not in joint)\n\n for joint in joints:\n r = quat_df[[c for c in quat_params if joint in c]] # Get the columns that belong to this joint\n \n euler_df.drop(['%s_qWrotation'%joint, '%s_qXrotation'%joint, '%s_qYrotation'%joint, '%s_qZrotation'%joint], axis=1, inplace=True)\n quat = [[f[1]['%s_qWrotation'%joint], f[1]['%s_qXrotation'%joint], f[1]['%s_qYrotation'%joint], f[1]['%s_qZrotation'%joint]] for f in r.iterrows()] # Make sure the columsn are organized in xyz order\n quats=Quaternions(np.asarray(quat))\n euler_rots = 180/np.pi*quats.euler()\n track.skeleton[joint]['order'] = 'ZYX'\n rot_order = track.skeleton[joint]['order']\n #euler_rots = [Rotation(f, 'expmap').to_euler(True, rot_order) for f in expmap] # Convert the exp maps to eulers\n #euler_rots = [expmap2euler(f, rot_order, True) for f in expmap] # Convert the exp maps to eulers\n \n # Create the corresponding columns in the new DataFrame\n \n euler_df['%s_%srotation'%(joint, rot_order[2])] = pd.Series(data=[e[0] for e in euler_rots], index=euler_df.index)\n euler_df['%s_%srotation'%(joint, rot_order[1])] = pd.Series(data=[e[1] for e in euler_rots], index=euler_df.index)\n euler_df['%s_%srotation'%(joint, rot_order[0])] = pd.Series(data=[e[2] for e in euler_rots], index=euler_df.index)\n\n new_track = track.clone()\n new_track.values = euler_df\n Q.append(new_track)\n\n return Q\n \n def _to_pos(self, X):\n '''Converts joints rotations in Euler angles to joint positions'''\n\n Q = []\n for track in X:\n channels = []\n titles = []\n euler_df = track.values\n\n # Create a new DataFrame to store the exponential map rep\n pos_df = pd.DataFrame(index=euler_df.index)\n\n # List the columns that contain rotation channels\n rot_cols = [c for c in euler_df.columns if ('rotation' in c)]\n\n # List the columns that contain position channels\n pos_cols = [c for c in euler_df.columns if ('position' in c)]\n\n # List the joints that are not end sites, i.e., have channels\n joints = (joint for joint in track.skeleton)\n \n tree_data = {}\n\n for joint in track.traverse():\n parent = track.skeleton[joint]['parent']\n rot_order = track.skeleton[joint]['order']\n #print(\"rot_order:\" + joint + \" :\" + rot_order)\n\n # Get the rotation columns that belong to this joint\n rc = euler_df[[c for c in rot_cols if joint in c]]\n\n # Get the position columns that belong to this joint\n pc = euler_df[[c for c in pos_cols if joint in c]]\n\n # Make sure the columns are organized in xyz order\n if rc.shape[1] < 3:\n euler_values = np.zeros((euler_df.shape[0], 3))\n rot_order = \"XYZ\"\n else:\n euler_values = np.pi/180.0*np.transpose(np.array([track.values['%s_%srotation'%(joint, rot_order[0])], track.values['%s_%srotation'%(joint, rot_order[1])], track.values['%s_%srotation'%(joint, rot_order[2])]]))\n\n if pc.shape[1] < 3:\n pos_values = np.asarray([[0,0,0] for f in pc.iterrows()])\n else:\n pos_values =np.asarray([[f[1]['%s_Xposition'%joint], \n f[1]['%s_Yposition'%joint], \n f[1]['%s_Zposition'%joint]] for f in pc.iterrows()])\n \n quats = Quaternions.from_euler(np.asarray(euler_values), order=rot_order.lower(), world=False)\n \n tree_data[joint]=[\n [], # to store the rotation matrix\n [] # to store the calculated position\n ] \n if track.root_name == joint:\n tree_data[joint][0] = quats#rotmats\n # tree_data[joint][1] = np.add(pos_values, track.skeleton[joint]['offsets'])\n tree_data[joint][1] = pos_values\n else:\n # for every frame i, multiply this joint's rotmat to the rotmat of its parent\n tree_data[joint][0] = tree_data[parent][0]*quats# np.matmul(rotmats, tree_data[parent][0])\n\n # add the position channel to the offset and store it in k, for every frame i\n k = pos_values + np.asarray(track.skeleton[joint]['offsets'])\n\n # multiply k to the rotmat of the parent for every frame i\n q = tree_data[parent][0]*k #np.matmul(k.reshape(k.shape[0],1,3), tree_data[parent][0])\n\n # add q to the position of the parent, for every frame i\n tree_data[joint][1] = tree_data[parent][1] + q #q.reshape(k.shape[0],3) + tree_data[parent][1]\n\n # Create the corresponding columns in the new DataFrame\n df = pd.DataFrame(data=tree_data[joint][1], \n index=pos_df.index, \n columns=['%s_Xposition'%joint, '%s_Yposition'%joint, '%s_Zposition'%joint])\n pos_df = pd.concat((pos_df, df), axis=1)\n\n new_track = track.clone()\n new_track.values = pos_df\n Q.append(new_track)\n return Q\n\n def _expmap2rot(self, expmap):\n\n theta = np.linalg.norm(expmap, axis=1, keepdims=True)\n nz = np.nonzero(theta)[0]\n\n expmap[nz,:] = expmap[nz,:]/theta[nz]\n\n nrows=expmap.shape[0]\n x = expmap[:,0]\n y = expmap[:,1]\n z = expmap[:,2]\n\n s = np.sin(theta*0.5).reshape(nrows)\n c = np.cos(theta*0.5).reshape(nrows)\n\n rotmats = np.zeros((nrows, 3, 3))\n\n rotmats[:,0,0] = 2*(x*x-1)*s*s+1\n rotmats[:,0,1] = 2*x*y*s*s-2*z*c*s\n rotmats[:,0,2] = 2*x*z*s*s+2*y*c*s\n rotmats[:,1,0] = 2*x*y*s*s+2*z*c*s\n rotmats[:,1,1] = 2*(y*y-1)*s*s+1\n rotmats[:,1,2] = 2*y*z*s*s-2*x*c*s\n rotmats[:,2,0] = 2*x*z*s*s-2*y*c*s\n rotmats[:,2,1] = 2*y*z*s*s+2*x*c*s\n rotmats[:,2,2] = 2*(z*z-1)*s*s+1\n\n return rotmats\n\n def _expmap_to_pos(self, X):\n '''Converts joints rotations in expmap notation to joint positions'''\n\n Q = []\n for track in X:\n channels = []\n titles = []\n exp_df = track.values\n\n # Create a new DataFrame to store the exponential map rep\n pos_df = pd.DataFrame(index=exp_df.index)\n\n # List the columns that contain rotation channels\n exp_params = [c for c in exp_df.columns if ( any(p in c for p in ['alpha', 'beta','gamma']) and 'Nub' not in c)]\n\n # List the joints that are not end sites, i.e., have channels\n joints = (joint for joint in track.skeleton)\n\n tree_data = {}\n \n for joint in track.traverse():\n parent = track.skeleton[joint]['parent']\n \n if 'Nub' not in joint:\n r = exp_df[[c for c in exp_params if joint in c]] # Get the columns that belong to this joint\n expmap = r.values\n #expmap = [[f[1]['%s_alpha'%joint], f[1]['%s_beta'%joint], f[1]['%s_gamma'%joint]] for f in r.iterrows()]\n else:\n expmap = np.zeros((exp_df.shape[0], 3))\n\n # Convert the eulers to rotation matrices\n #rotmats = np.asarray([Rotation(f, 'expmap').rotmat for f in expmap])\n #angs = np.linalg.norm(expmap,axis=1, keepdims=True)\n rotmats = self._expmap2rot(expmap)\n \n tree_data[joint]=[\n [], # to store the rotation matrix\n [] # to store the calculated position\n ] \n pos_values = np.zeros((exp_df.shape[0], 3))\n\n if track.root_name == joint:\n tree_data[joint][0] = rotmats\n # tree_data[joint][1] = np.add(pos_values, track.skeleton[joint]['offsets'])\n tree_data[joint][1] = pos_values\n else:\n # for every frame i, multiply this joint's rotmat to the rotmat of its parent\n tree_data[joint][0] = np.matmul(rotmats, tree_data[parent][0])\n\n # add the position channel to the offset and store it in k, for every frame i\n k = pos_values + track.skeleton[joint]['offsets']\n\n # multiply k to the rotmat of the parent for every frame i\n q = np.matmul(k.reshape(k.shape[0],1,3), tree_data[parent][0])\n\n # add q to the position of the parent, for every frame i\n tree_data[joint][1] = q.reshape(k.shape[0],3) + tree_data[parent][1]\n\n\n # Create the corresponding columns in the new DataFrame\n df = pd.DataFrame(data=tree_data[joint][1], \n index=pos_df.index, \n columns=['%s_Xposition'%joint, '%s_Yposition'%joint, '%s_Zposition'%joint])\n pos_df = pd.concat((pos_df, df), axis=1)\n\n new_track = track.clone()\n new_track.values = pos_df\n Q.append(new_track)\n return Q\n\n def _to_expmap(self, X):\n '''Converts Euler angles to Exponential Maps'''\n\n Q = []\n for track in X:\n channels = []\n titles = []\n euler_df = track.values\n\n # Create a new DataFrame to store the exponential map rep\n exp_df = euler_df.copy()# pd.DataFrame(index=euler_df.index)\n\n # List the columns that contain rotation channels\n rots = [c for c in euler_df.columns if ('rotation' in c and 'Nub' not in c)]\n\n # List the joints that are not end sites, i.e., have channels\n joints = (joint for joint in track.skeleton if 'Nub' not in joint)\n\n for joint in joints:\n #print(joint)\n r = euler_df[[c for c in rots if joint in c]] # Get the columns that belong to this joint\n rot_order = track.skeleton[joint]['order']\n r1_col = '%s_%srotation'%(joint, rot_order[0])\n r2_col = '%s_%srotation'%(joint, rot_order[1])\n r3_col = '%s_%srotation'%(joint, rot_order[2])\n \n exp_df.drop([r1_col, r2_col, r3_col], axis=1, inplace=True)\n euler = np.transpose(np.array([r[r1_col], r[r2_col], r[r3_col]]))\n #exps = [Rotation(f, 'euler', from_deg=True, order=rot_order).to_expmap() for f in euler] # Convert the eulers to exp maps\n exps = unroll(np.array([euler2expmap(f, rot_order, True) for f in euler])) # Convert the exp maps to eulers\n #exps = euler2expmap2(euler, rot_order, True) # Convert the eulers to exp maps\n\n # Create the corresponding columns in the new DataFrame \n exp_df.insert(loc=0, column='%s_gamma'%joint, value=pd.Series(data=[e[2] for e in exps], index=exp_df.index))\n exp_df.insert(loc=0, column='%s_beta'%joint, value=pd.Series(data=[e[1] for e in exps], index=exp_df.index))\n exp_df.insert(loc=0, column='%s_alpha'%joint, value=pd.Series(data=[e[0] for e in exps], index=exp_df.index))\n\n #print(exp_df.columns)\n new_track = track.clone()\n new_track.values = exp_df\n Q.append(new_track)\n\n return Q\n \n def _expmap_to_euler(self, X):\n Q = []\n for track in X:\n channels = []\n titles = []\n exp_df = track.values\n\n # Create a new DataFrame to store the exponential map rep\n euler_df = exp_df.copy()\n \n # List the columns that contain rotation channels\n exp_params = [c for c in exp_df.columns if ( any(p in c for p in ['alpha', 'beta','gamma']) and 'Nub' not in c)]\n\n # List the joints that are not end sites, i.e., have channels\n joints = (joint for joint in track.skeleton if 'Nub' not in joint)\n\n for joint in joints:\n r = exp_df[[c for c in exp_params if joint in c]] # Get the columns that belong to this joint\n jt_alpha = '%s_alpha'%joint\n jt_beta = '%s_beta'%joint\n jt_gamma = '%s_gamma'%joint\n \n euler_df.drop([jt_alpha, jt_beta, jt_gamma], axis=1, inplace=True)\n expmap = np.transpose(np.array([track.values[jt_alpha], track.values[jt_beta], track.values[jt_gamma]]))\n rot_order = track.skeleton[joint]['order']\n euler_rots = np.array(R.from_rotvec(expmap).as_euler(rot_order, degrees=True)) \n #euler_rots = [expmap2euler(f, rot_order, True) for f in expmap] # Convert the exp maps to eulers\n \n # Create the corresponding columns in the new DataFrame \n euler_df['%s_%srotation'%(joint, rot_order[0])] = pd.Series(data=[e[0] for e in euler_rots], index=euler_df.index)\n euler_df['%s_%srotation'%(joint, rot_order[1])] = pd.Series(data=[e[1] for e in euler_rots], index=euler_df.index)\n euler_df['%s_%srotation'%(joint, rot_order[2])] = pd.Series(data=[e[2] for e in euler_rots], index=euler_df.index)\n\n new_track = track.clone()\n new_track.values = euler_df\n Q.append(new_track)\n\n return Q\n\n def _to_expmap2(self, X):\n '''Converts Euler angles to Exponential Maps'''\n\n Q = []\n for track in X:\n channels = []\n titles = []\n euler_df = track.values\n\n # Create a new DataFrame to store the exponential map rep\n exp_df = euler_df.copy()# pd.DataFrame(index=euler_df.index)\n\n # Copy the root positions into the new DataFrame\n #rxp = '%s_Xposition'%track.root_name\n #ryp = '%s_Yposition'%track.root_name\n #rzp = '%s_Zposition'%track.root_name\n #exp_df[rxp] = pd.Series(data=euler_df[rxp], index=exp_df.index)\n #exp_df[ryp] = pd.Series(data=euler_df[ryp], index=exp_df.index)\n #exp_df[rzp] = pd.Series(data=euler_df[rzp], index=exp_df.index)\n\n # List the columns that contain rotation channels\n rots = [c for c in euler_df.columns if ('rotation' in c and 'Nub' not in c)]\n\n # List the joints that are not end sites, i.e., have channels\n joints = (joint for joint in track.skeleton if 'Nub' not in joint)\n\n for joint in joints:\n r = euler_df[[c for c in rots if joint in c]] # Get the columns that belong to this joint\n rot_order = track.skeleton[joint]['order']\n\n # Get the rotation columns that belong to this joint\n rc = euler_df[[c for c in rots if joint in c]]\n\n r1_col = '%s_%srotation'%(joint, rot_order[0])\n r2_col = '%s_%srotation'%(joint, rot_order[1])\n r3_col = '%s_%srotation'%(joint, rot_order[2])\n # Make sure the columns are organized in xyz order\n #print(\"joint:\" + str(joint) + \" rot_order:\" + str(rot_order))\n if rc.shape[1] < 3:\n euler_values = np.zeros((euler_df.shape[0], 3))\n rot_order = \"XYZ\"\n else:\n euler_values = np.pi/180.0*np.transpose(np.array([track.values[r1_col], track.values[r2_col], track.values[r3_col]]))\n \n quats = Quaternions.from_euler(np.asarray(euler_values), order=rot_order.lower(), world=False)\n #exps = [Rotation(f, 'euler', from_deg=True, order=rot_order).to_expmap() for f in euler] # Convert the eulers to exp maps\n #exps = unroll(np.array([euler2expmap(f, rot_order, True) for f in euler])) # Convert the exp maps to eulers\n #exps = euler2expmap2(euler, rot_order, True) # Convert the eulers to exp maps\n # Create the corresponding columns in the new DataFrame\n if (self.ref_pose is not None):\n q1_col = '%s_qWrotation'%(joint)\n q2_col = '%s_qXrotation'%(joint)\n q3_col = '%s_qYrotation'%(joint)\n q4_col = '%s_qZrotation'%(joint)\n ref_q = Quaternions(np.asarray([[f[1][q1_col], f[1][q2_col], f[1][q3_col], f[1][q4_col]] for f in self.ref_pose.values.iterrows()]))\n #print(\"ref_q:\" + str(ref_q.shape))\n ref_q = ref_q[0,:]\n quats=(-ref_q)*quats\n \n angles, axis = quats.angle_axis()\n aa = np.where(angles>np.pi)\n angles[aa] = angles[aa]-2*np.pi \n #exps = unroll(angles[:,None]*axis)\n exps = angles[:,None]*axis\n #print(f\"{joint}: {str(exps[0,:])}\")\n\n #exps = np.array([quat2expmap(f) for f in quats])\n exp_df.drop([r1_col, r2_col, r3_col], axis=1, inplace=True)\n exp_df.insert(loc=0, column='%s_gamma'%joint, value=pd.Series(data=[e[2] for e in exps], index=exp_df.index))\n exp_df.insert(loc=0, column='%s_beta'%joint, value=pd.Series(data=[e[1] for e in exps], index=exp_df.index))\n exp_df.insert(loc=0, column='%s_alpha'%joint, value=pd.Series(data=[e[0] for e in exps], index=exp_df.index))\n\n #print(exp_df.columns)\n new_track = track.clone()\n new_track.values = exp_df\n Q.append(new_track)\n\n return Q\n \n def _expmap_to_euler2(self, X):\n Q = []\n for track in X:\n channels = []\n titles = []\n exp_df = track.values\n\n # Create a new DataFrame to store the exponential map rep\n #euler_df = pd.DataFrame(index=exp_df.index)\n euler_df = exp_df.copy()\n\n # Copy the root positions into the new DataFrame\n #rxp = '%s_Xposition'%track.root_name\n #ryp = '%s_Yposition'%track.root_name\n #rzp = '%s_Zposition'%track.root_name\n #euler_df[rxp] = pd.Series(data=exp_df[rxp], index=euler_df.index)\n #euler_df[ryp] = pd.Series(data=exp_df[ryp], index=euler_df.index)\n #euler_df[rzp] = pd.Series(data=exp_df[rzp], index=euler_df.index)\n \n # List the columns that contain rotation channels\n exp_params = [c for c in exp_df.columns if ( any(p in c for p in ['alpha', 'beta','gamma']) and 'Nub' not in c)]\n\n # List the joints that are not end sites, i.e., have channels\n joints = (joint for joint in track.skeleton if 'Nub' not in joint)\n\n for joint in joints:\n r = exp_df[[c for c in exp_params if joint in c]] # Get the columns that belong to this joint\n \n euler_df.drop(['%s_alpha'%joint, '%s_beta'%joint, '%s_gamma'%joint], axis=1, inplace=True)\n expmap = [[f[1]['%s_alpha'%joint], f[1]['%s_beta'%joint], f[1]['%s_gamma'%joint]] for f in r.iterrows()] # Make sure the columsn are organized in xyz order\n angs = np.linalg.norm(expmap, axis=1)\n quats=Quaternions.from_angle_axis(angs, expmap/(np.tile(angs[:, None]+1e-10, (1,3))))\n if (self.ref_pose is not None):\n q1_col = '%s_qWrotation'%(joint)\n q2_col = '%s_qXrotation'%(joint)\n q3_col = '%s_qYrotation'%(joint)\n q4_col = '%s_qZrotation'%(joint)\n ref_q = Quaternions(np.asarray([[f[1][q1_col], f[1][q2_col], f[1][q3_col], f[1][q4_col]] for f in self.ref_pose.values.iterrows()]))\n #print(\"ref_q:\" + str(ref_q.shape))\n ref_q = ref_q[0,:]\n quats=ref_q*quats\n \n euler_rots = 180/np.pi*quats.euler()\n track.skeleton[joint]['order'] = 'ZYX'\n rot_order = track.skeleton[joint]['order']\n #euler_rots = [Rotation(f, 'expmap').to_euler(True, rot_order) for f in expmap] # Convert the exp maps to eulers\n #euler_rots = [expmap2euler(f, rot_order, True) for f in expmap] # Convert the exp maps to eulers\n \n # Create the corresponding columns in the new DataFrame\n \n euler_df['%s_%srotation'%(joint, rot_order[2])] = pd.Series(data=[e[0] for e in euler_rots], index=euler_df.index)\n euler_df['%s_%srotation'%(joint, rot_order[1])] = pd.Series(data=[e[1] for e in euler_rots], index=euler_df.index)\n euler_df['%s_%srotation'%(joint, rot_order[0])] = pd.Series(data=[e[2] for e in euler_rots], index=euler_df.index)\n\n new_track = track.clone()\n new_track.values = euler_df\n Q.append(new_track)\n\n return Q\n \n def _euler_to_vectors(self, X):\n '''Converts Euler angles to Up and Fwd vectors'''\n\n Q = []\n for track in X:\n channels = []\n titles = []\n euler_df = track.values\n\n # Create a new DataFrame to store the exponential map rep\n vec_df = euler_df.copy()# pd.DataFrame(index=euler_df.index)\n\n # List the columns that contain rotation channels\n rots = [c for c in euler_df.columns if ('rotation' in c and 'Nub' not in c)]\n\n # List the joints that are not end sites, i.e., have channels\n joints = (joint for joint in track.skeleton if 'Nub' not in joint)\n\n for joint in joints:\n #print(joint)\n r = euler_df[[c for c in rots if joint in c]] # Get the columns that belong to this joint\n rot_order = track.skeleton[joint]['order']\n r1_col = '%s_%srotation'%(joint, rot_order[0])\n r2_col = '%s_%srotation'%(joint, rot_order[1])\n r3_col = '%s_%srotation'%(joint, rot_order[2])\n \n vec_df.drop([r1_col, r2_col, r3_col], axis=1, inplace=True)\n euler = [[f[1][r1_col], f[1][r2_col], f[1][r3_col]] for f in r.iterrows()]\n vectors = np.array([euler2vectors(f, rot_order, True) for f in euler])\n \n vec_df.insert(loc=0, column='%s_xUp'%joint, value=pd.Series(data=[e[0] for e in vectors], index=vec_df.index))\n vec_df.insert(loc=0, column='%s_yUp'%joint, value=pd.Series(data=[e[1] for e in vectors], index=vec_df.index))\n vec_df.insert(loc=0, column='%s_zUp'%joint, value=pd.Series(data=[e[2] for e in vectors], index=vec_df.index))\n vec_df.insert(loc=0, column='%s_xFwd'%joint, value=pd.Series(data=[e[3] for e in vectors], index=vec_df.index))\n vec_df.insert(loc=0, column='%s_yFwd'%joint, value=pd.Series(data=[e[4] for e in vectors], index=vec_df.index))\n vec_df.insert(loc=0, column='%s_zFwd'%joint, value=pd.Series(data=[e[5] for e in vectors], index=vec_df.index))\n\n #print(exp_df.columns)\n new_track = track.clone()\n new_track.values = vec_df\n Q.append(new_track)\n\n return Q\n \n def _vectors_to_euler(self, X):\n '''Converts Up and Fwd vectors to Euler angles'''\n Q = []\n for track in X:\n channels = []\n titles = []\n vec_df = track.values\n\n # Create a new DataFrame to store the exponential map rep\n #euler_df = pd.DataFrame(index=exp_df.index)\n euler_df = vec_df.copy()\n\n # List the columns that contain rotation channels\n vec_params = [c for c in vec_df.columns if ( any(p in c for p in ['xUp', 'yUp','zUp','xFwd', 'yFwd','zFwd']) and 'Nub' not in c)]\n\n # List the joints that are not end sites, i.e., have channels\n joints = (joint for joint in track.skeleton if 'Nub' not in joint)\n\n for joint in joints:\n r = vec_df[[c for c in vec_params if joint in c]] # Get the columns that belong to this joint\n \n euler_df.drop(['%s_xUp'%joint, '%s_yUp'%joint, '%s_zUp'%joint, '%s_xFwd'%joint, '%s_yFwd'%joint, '%s_zFwd'%joint], axis=1, inplace=True)\n vectors = [[f[1]['%s_xUp'%joint], f[1]['%s_yUp'%joint], f[1]['%s_zUp'%joint], f[1]['%s_xFwd'%joint], f[1]['%s_yFwd'%joint], f[1]['%s_zFwd'%joint]] for f in r.iterrows()] # Make sure the columsn are organized in xyz order\n rot_order = track.skeleton[joint]['order']\n euler_rots = [vectors2euler(f, rot_order, True) for f in vectors]\n \n # Create the corresponding columns in the new DataFrame\n \n euler_df['%s_%srotation'%(joint, rot_order[0])] = pd.Series(data=[e[0] for e in euler_rots], index=euler_df.index)\n euler_df['%s_%srotation'%(joint, rot_order[1])] = pd.Series(data=[e[1] for e in euler_rots], index=euler_df.index)\n euler_df['%s_%srotation'%(joint, rot_order[2])] = pd.Series(data=[e[2] for e in euler_rots], index=euler_df.index)\n\n new_track = track.clone()\n new_track.values = euler_df\n Q.append(new_track)\n\n return Q\n\nclass Mirror(BaseEstimator, TransformerMixin):\n def __init__(self, axis=\"X\", append=True):\n \"\"\"\n Mirrors the data \n \"\"\"\n self.axis = axis\n self.append = append\n \n \n def fit(self, X, y=None):\n return self\n \n def transform(self, X, y=None):\n #print(\"Mirror: \" + self.axis)\n Q = []\n \n if self.append:\n for track in X:\n Q.append(track)\n \n for track in X:\n channels = []\n titles = []\n \n if self.axis == \"X\":\n signs = np.array([1,-1,-1])\n if self.axis == \"Y\":\n signs = np.array([-1,1,-1])\n if self.axis == \"Z\":\n signs = np.array([-1,-1,1])\n\n euler_df = track.values\n\n # Create a new DataFrame to store the exponential map rep\n new_df = pd.DataFrame(index=euler_df.index)\n\n # Copy the root positions into the new DataFrame\n rxp = '%s_Xposition'%track.root_name\n ryp = '%s_Yposition'%track.root_name\n rzp = '%s_Zposition'%track.root_name\n new_df[rxp] = pd.Series(data=-signs[0]*euler_df[rxp], index=new_df.index)\n new_df[ryp] = pd.Series(data=-signs[1]*euler_df[ryp], index=new_df.index)\n new_df[rzp] = pd.Series(data=-signs[2]*euler_df[rzp], index=new_df.index)\n \n # List the columns that contain rotation channels\n rots = [c for c in euler_df.columns if ('rotation' in c and 'Nub' not in c)]\n #lft_rots = [c for c in euler_df.columns if ('Left' in c and 'rotation' in c and 'Nub' not in c)]\n #rgt_rots = [c for c in euler_df.columns if ('Right' in c and 'rotation' in c and 'Nub' not in c)]\n lft_joints = (joint for joint in track.skeleton if 'Left' in joint and 'Nub' not in joint)\n rgt_joints = (joint for joint in track.skeleton if 'Right' in joint and 'Nub' not in joint)\n \n new_track = track.clone()\n\n for lft_joint in lft_joints:\n #lr = euler_df[[c for c in rots if lft_joint + \"_\" in c]] \n #rot_order = track.skeleton[lft_joint]['order'] \n #lft_eulers = [[f[1]['%s_Xrotation'%lft_joint], f[1]['%s_Yrotation'%lft_joint], f[1]['%s_Zrotation'%lft_joint]] for f in lr.iterrows()]\n \n rgt_joint = lft_joint.replace('Left', 'Right')\n #rr = euler_df[[c for c in rots if rgt_joint + \"_\" in c]]\n #rot_order = track.skeleton[rgt_joint]['order'] \n# rgt_eulers = [[f[1]['%s_Xrotation'%rgt_joint], f[1]['%s_Yrotation'%rgt_joint], f[1]['%s_Zrotation'%rgt_joint]] for f in rr.iterrows()]\n \n # Create the corresponding columns in the new DataFrame\n \n new_df['%s_Xrotation'%lft_joint] = pd.Series(data=signs[0]*track.values['%s_Xrotation'%rgt_joint], index=new_df.index)\n new_df['%s_Yrotation'%lft_joint] = pd.Series(data=signs[1]*track.values['%s_Yrotation'%rgt_joint], index=new_df.index)\n new_df['%s_Zrotation'%lft_joint] = pd.Series(data=signs[2]*track.values['%s_Zrotation'%rgt_joint], index=new_df.index)\n \n new_df['%s_Xrotation'%rgt_joint] = pd.Series(data=signs[0]*track.values['%s_Xrotation'%lft_joint], index=new_df.index)\n new_df['%s_Yrotation'%rgt_joint] = pd.Series(data=signs[1]*track.values['%s_Yrotation'%lft_joint], index=new_df.index)\n new_df['%s_Zrotation'%rgt_joint] = pd.Series(data=signs[2]*track.values['%s_Zrotation'%lft_joint], index=new_df.index)\n \n # List the joints that are not left or right, i.e. are on the trunk\n joints = (joint for joint in track.skeleton if 'Nub' not in joint and 'Left' not in joint and 'Right' not in joint)\n\n for joint in joints:\n #r = euler_df[[c for c in rots if joint in c]] # Get the columns that belong to this joint\n #rot_order = track.skeleton[joint]['order']\n\n #eulers = [[f[1]['%s_Xrotation'%joint], f[1]['%s_Yrotation'%joint], f[1]['%s_Zrotation'%joint]] for f in r.iterrows()]\n\n # Create the corresponding columns in the new DataFrame\n new_df['%s_Xrotation'%joint] = pd.Series(data=signs[0]*track.values['%s_Xrotation'%joint], index=new_df.index)\n new_df['%s_Yrotation'%joint] = pd.Series(data=signs[1]*track.values['%s_Yrotation'%joint], index=new_df.index)\n new_df['%s_Zrotation'%joint] = pd.Series(data=signs[2]*track.values['%s_Zrotation'%joint], index=new_df.index)\n\n new_track.values = new_df\n new_track.take_name = track.take_name + \"_mirrored\"\n Q.append(new_track)\n\n return Q\n\n def inverse_transform(self, X, copy=None, start_pos=None):\n return X\n\nclass EulerReorder(BaseEstimator, TransformerMixin):\n def __init__(self, new_order):\n \"\"\"\n Add a \n \"\"\"\n self.new_order = new_order\n \n \n def fit(self, X, y=None):\n self.orig_skeleton = copy.deepcopy(X[0].skeleton)\n return self\n \n def transform(self, X, y=None):\n #print(\"EulerReorder\")\n Q = []\n\n for track in X:\n channels = []\n titles = []\n euler_df = track.values\n\n # Create a new DataFrame to store the exponential map rep\n #new_df = pd.DataFrame(index=euler_df.index)\n new_df = euler_df.copy()\n\n # Copy the root positions into the new DataFrame\n rxp = '%s_Xposition'%track.root_name\n ryp = '%s_Yposition'%track.root_name\n rzp = '%s_Zposition'%track.root_name\n new_df[rxp] = pd.Series(data=euler_df[rxp], index=new_df.index)\n new_df[ryp] = pd.Series(data=euler_df[ryp], index=new_df.index)\n new_df[rzp] = pd.Series(data=euler_df[rzp], index=new_df.index)\n \n # List the columns that contain rotation channels\n rots = [c for c in euler_df.columns if ('rotation' in c and 'Nub' not in c)]\n\n # List the joints that are not end sites, i.e., have channels\n joints = (joint for joint in track.skeleton if 'Nub' not in joint)\n\n new_track = track.clone()\n for joint in joints:\n r = euler_df[[c for c in rots if joint in c]] # Get the columns that belong to this joint\n rot_order = track.skeleton[joint]['order']\n r1_col = '%s_%srotation'%(joint, rot_order[0])\n r2_col = '%s_%srotation'%(joint, rot_order[1])\n r3_col = '%s_%srotation'%(joint, rot_order[2])\n \n #euler = [[f[1][r1_col], f[1][r2_col], f[1][r3_col]] for f in r.iterrows()]\n euler = np.transpose(np.array([r[r1_col], r[r2_col], r[r3_col]])) \n \n #euler = [[f[1]['%s_Xrotation'%(joint)], f[1]['%s_Yrotation'%(joint)], f[1]['%s_Zrotation'%(joint)]] for f in r.iterrows()]\n new_euler = [euler_reorder(f, rot_order, self.new_order, True) for f in euler]\n #new_euler = euler_reorder2(np.array(euler), rot_order, self.new_order, True)\n \n # Create the corresponding columns in the new DataFrame\n new_df['%s_%srotation'%(joint, self.new_order[0])] = pd.Series(data=[e[0] for e in new_euler], index=new_df.index)\n new_df['%s_%srotation'%(joint, self.new_order[1])] = pd.Series(data=[e[1] for e in new_euler], index=new_df.index)\n new_df['%s_%srotation'%(joint, self.new_order[2])] = pd.Series(data=[e[2] for e in new_euler], index=new_df.index)\n \n new_track.skeleton[joint]['order'] = self.new_order\n\n new_track.values = new_df\n Q.append(new_track)\n\n return Q\n \n def inverse_transform(self, X, copy=None, start_pos=None):\n return X\n\nclass JointSelector(BaseEstimator, TransformerMixin):\n '''\n Allows for filtering the mocap data to include only the selected joints\n '''\n def __init__(self, joints, include_root=False):\n self.joints = joints\n self.include_root = include_root\n\n def fit(self, X, y=None):\n selected_joints = []\n selected_channels = []\n\n if self.include_root:\n selected_joints.append(X[0].root_name)\n \n selected_joints.extend(self.joints)\n\n for joint_name in selected_joints:\n if joint_name.endswith(\"_Nub\"):\n selected_channels.extend([o for o in X[0].values.columns if (joint_name + \"_\") in o])\n else:\n selected_channels.extend([o for o in X[0].values.columns if (joint_name + \"_\") in o and 'Nub' not in o])\n \n self.selected_joints = selected_joints\n self.selected_channels = selected_channels\n self.not_selected = X[0].values.columns.difference(selected_channels)\n self.not_selected_values = {c:X[0].values[c].values[0] for c in self.not_selected}\n\n self.orig_skeleton = X[0].skeleton\n return self\n\n def transform(self, X, y=None):\n #print(\"JointSelector\")\n Q = []\n for track in X:\n t2 = track.clone()\n for key in track.skeleton.keys():\n if key not in self.selected_joints:\n t2.skeleton.pop(key)\n t2.values = track.values[self.selected_channels]\n\n for key in t2.skeleton.keys():\n to_remove = list(set(t2.skeleton[key]['children']) - set(self.selected_joints))\n [t2.skeleton[key]['children'].remove(c) for c in to_remove]\n\n Q.append(t2)\n \n\n return Q\n \n def inverse_transform(self, X, copy=None):\n Q = []\n\n for track in X:\n t2 = track.clone()\n skeleton = self.orig_skeleton\n for key in track.skeleton.keys():\n skeleton[key]['order']=track.skeleton[key]['order'] \n \n t2.skeleton = skeleton\n for d in self.not_selected:\n t2.values[d] = self.not_selected_values[d]\n Q.append(t2)\n\n return Q\n\n\nclass Numpyfier(BaseEstimator, TransformerMixin):\n '''\n Just converts the values in a MocapData object into a numpy array\n Useful for the final stage of a pipeline before training\n '''\n def __init__(self):\n pass\n\n def fit(self, X, y=None):\n self.org_mocap_ = X[0].clone()\n self.org_mocap_.values.drop(self.org_mocap_.values.index, inplace=True)\n\n return self\n\n def transform(self, X, y=None):\n #print(\"Numpyfier\")\n Q = []\n \n for track in X:\n Q.append(track.values.values)\n #print(\"Numpyfier:\" + str(track.values.columns))\n \n return np.array(Q)\n\n def inverse_transform(self, X, copy=None):\n Q = []\n\n for track in X:\n new_mocap = self.org_mocap_.clone()\n time_index = pd.to_timedelta([f for f in range(track.shape[0])], unit='s')*self.org_mocap_.framerate\n\n new_df = pd.DataFrame(data=track, index=time_index, columns=self.org_mocap_.values.columns)\n \n new_mocap.values = new_df\n \n\n Q.append(new_mocap)\n\n return Q\n \nclass Slicer(BaseEstimator, TransformerMixin):\n '''\n Slice the data into intervals of equal size \n '''\n def __init__(self, window_size, overlap=0.5):\n self.window_size = window_size\n self.overlap = overlap\n pass\n\n def fit(self, X, y=None):\n self.org_mocap_ = X[0].clone()\n self.org_mocap_.values.drop(self.org_mocap_.values.index, inplace=True)\n\n return self\n\n def transform(self, X, y=None):\n #print(\"Slicer\")\n Q = []\n \n for track in X:\n vals = track.values.values\n nframes = vals.shape[0]\n overlap_frames = (int)(self.overlap*self.window_size)\n \n n_sequences = (nframes-overlap_frames)//(self.window_size-overlap_frames)\n \n if n_sequences>0:\n y = np.zeros((n_sequences, self.window_size, vals.shape[1]))\n\n # extract sequences from the input data\n for i in range(0,n_sequences):\n frameIdx = (self.window_size-overlap_frames) * i\n Q.append(vals[frameIdx:frameIdx+self.window_size,:])\n\n return np.array(Q)\n\n def inverse_transform(self, X, copy=None):\n Q = []\n\n for track in X:\n \n new_mocap = self.org_mocap_.clone()\n time_index = pd.to_timedelta([f for f in range(track.shape[0])], unit='s')\n\n new_df = pd.DataFrame(data=track, index=time_index, columns=self.org_mocap_.values.columns)\n \n new_mocap.values = new_df\n \n\n Q.append(new_mocap)\n\n return Q\n\nclass RootTransformer(BaseEstimator, TransformerMixin):\n def __init__(self, method, hips_axis_order=\"XYZ\", position_smoothing=0, rotation_smoothing=0, separate_root=True):\n \"\"\"\n Accepted methods:\n abdolute_translation_deltas\n pos_rot_deltas\n \"\"\"\n self.method = method\n self.position_smoothing=position_smoothing\n self.rotation_smoothing=rotation_smoothing\n self.separate_root = separate_root\n self.hips_axis_order = hips_axis_order\n \n # relative rotation from the hips awis the the x-side, y-up, z-forward convention\n rot_mat = np.zeros((3,3))\n for i in range(3):\n ax_i = ord(hips_axis_order[i])-ord(\"X\") \n rot_mat[i,ax_i]=1\n self.root_rotation_offset = Quaternions.from_transforms(rot_mat[np.newaxis, :, :])\n self.hips_side_axis = -rot_mat[0,:]\n \n def fit(self, X, y=None):\n return self\n \n def transform(self, X, y=None):\n #print(\"RootTransformer\")\n Q = []\n\n for track in X:\n if self.method == 'abdolute_translation_deltas':\n new_df = track.values.copy()\n xpcol = '%s_Xposition'%track.root_name\n ypcol = '%s_Yposition'%track.root_name\n zpcol = '%s_Zposition'%track.root_name\n\n\n dxpcol = '%s_dXposition'%track.root_name\n dzpcol = '%s_dZposition'%track.root_name\n \n x=track.values[xpcol].copy()\n z=track.values[zpcol].copy()\n \n if self.position_smoothing>0:\n x_sm = filters.gaussian_filter1d(x, self.position_smoothing, axis=0, mode='nearest') \n z_sm = filters.gaussian_filter1d(z, self.position_smoothing, axis=0, mode='nearest') \n dx = pd.Series(data=x_sm, index=new_df.index).diff()\n dz = pd.Series(data=z_sm, index=new_df.index).diff()\n new_df[xpcol] = x-x_sm\n new_df[zpcol] = z-z_sm\n else:\n dx = x.diff()\n dz = z.diff()\n new_df.drop([xpcol, zpcol], axis=1, inplace=True)\n \n dx[0] = dx[1]\n dz[0] = dz[1]\n \n new_df[dxpcol] = dx\n new_df[dzpcol] = dz\n \n new_track = track.clone()\n new_track.values = new_df\n # end of abdolute_translation_deltas\n \n elif self.method == 'pos_rot_deltas':\n new_track = track.clone()\n\n # Absolute columns\n xp_col = '%s_Xposition'%track.root_name\n yp_col = '%s_Yposition'%track.root_name\n zp_col = '%s_Zposition'%track.root_name\n \n #rot_order = track.skeleton[track.root_name]['order']\n #%(joint, rot_order[0])\n\n rot_order = track.skeleton[track.root_name]['order']\n r1_col = '%s_%srotation'%(track.root_name, rot_order[0])\n r2_col = '%s_%srotation'%(track.root_name, rot_order[1])\n r3_col = '%s_%srotation'%(track.root_name, rot_order[2])\n\n # Delta columns\n # dxp_col = '%s_dXposition'%track.root_name\n # dzp_col = '%s_dZposition'%track.root_name\n\n # dxr_col = '%s_dXrotation'%track.root_name\n # dyr_col = '%s_dYrotation'%track.root_name\n # dzr_col = '%s_dZrotation'%track.root_name\n dxp_col = 'reference_dXposition'\n dzp_col = 'reference_dZposition'\n dxr_col = 'reference_dXrotation'\n dyr_col = 'reference_dYrotation'\n dzr_col = 'reference_dZrotation'\n\n positions = np.transpose(np.array([track.values[xp_col], track.values[yp_col], track.values[zp_col]]))\n rotations = np.pi/180.0*np.transpose(np.array([track.values[r1_col], track.values[r2_col], track.values[r3_col]]))\n \n \"\"\" Get Trajectory and smooth it\"\"\" \n trajectory_filterwidth = self.position_smoothing\n reference = positions.copy()*np.array([1,0,1])\n if trajectory_filterwidth>0:\n reference = filters.gaussian_filter1d(reference, trajectory_filterwidth, axis=0, mode='nearest')\n \n \"\"\" Get Root Velocity \"\"\"\n velocity = np.diff(reference, axis=0) \n velocity = np.vstack((velocity[0,:], velocity))\n\n \"\"\" Remove Root Translation \"\"\"\n positions = positions-reference\n\n \"\"\" Get Forward Direction along the x-z plane, assuming character is facig z-forward \"\"\"\n #forward = [Rotation(f, 'euler', from_deg=True, order=rot_order).rotmat[:,2] for f in rotations] # get the z-axis of the rotation matrix, assuming character is facig z-forward\n #print(\"order:\" + rot_order.lower())\n quats = Quaternions.from_euler(rotations, order=rot_order.lower(), world=False)\n #forward = quats*np.array([[0,0,1]])\n #forward[:,1] = 0\n side_dirs = quats*self.hips_side_axis\n forward = np.cross(np.array([[0,1,0]]), side_dirs)\n\n \"\"\" Smooth Forward Direction \"\"\" \n direction_filterwidth = self.rotation_smoothing\n if direction_filterwidth>0:\n forward = filters.gaussian_filter1d(forward, direction_filterwidth, axis=0, mode='nearest') \n\n forward = forward / np.sqrt((forward**2).sum(axis=-1))[...,np.newaxis]\n\n \"\"\" Remove Y Rotation \"\"\"\n target = np.array([[0,0,1]]).repeat(len(forward), axis=0)\n rotation = Quaternions.between(target, forward)[:,np.newaxis] \n positions = (-rotation[:,0]) * positions\n #new_rotations = (-rotation[:,0]) * quats\n new_rotations = (-self.root_rotation_offset) * (-rotation[:,0]) * quats\n\n \"\"\" Get Root Rotation \"\"\"\n #print(rotation[:,0])\n velocity = (-rotation[:,0]) * velocity\n rvelocity = Pivots.from_quaternions(rotation[1:] * -rotation[:-1]).ps\n rvelocity = np.vstack((rvelocity[0], rvelocity))\n\n eulers = np.array([t3d.euler.quat2euler(q, axes=('s'+rot_order.lower()[::-1]))[::-1] for q in new_rotations])*180.0/np.pi\n \n new_df = track.values.copy()\n\n root_pos_x = pd.Series(data=positions[:,0], index=new_df.index)\n root_pos_y = pd.Series(data=positions[:,1], index=new_df.index)\n root_pos_z = pd.Series(data=positions[:,2], index=new_df.index)\n root_pos_x_diff = pd.Series(data=velocity[:,0], index=new_df.index)\n root_pos_z_diff = pd.Series(data=velocity[:,2], index=new_df.index)\n\n root_rot_1 = pd.Series(data=eulers[:,0], index=new_df.index)\n root_rot_2 = pd.Series(data=eulers[:,1], index=new_df.index)\n root_rot_3 = pd.Series(data=eulers[:,2], index=new_df.index)\n root_rot_y_diff = pd.Series(data=rvelocity[:,0], index=new_df.index)\n \n #new_df.drop([xr_col, yr_col, zr_col, xp_col, zp_col], axis=1, inplace=True)\n\n new_df[xp_col] = root_pos_x\n new_df[yp_col] = root_pos_y\n new_df[zp_col] = root_pos_z\n new_df[dxp_col] = root_pos_x_diff\n new_df[dzp_col] = root_pos_z_diff\n\n new_df[r1_col] = root_rot_1\n new_df[r2_col] = root_rot_2\n new_df[r3_col] = root_rot_3\n #new_df[dxr_col] = root_rot_x_diff\n new_df[dyr_col] = root_rot_y_diff\n #new_df[dzr_col] = root_rot_z_diff\n\n new_track.values = new_df\n elif self.method == 'pos_xyz_rot_deltas':\n new_track = track.clone()\n\n # Absolute columns\n xp_col = '%s_Xposition'%track.root_name\n yp_col = '%s_Yposition'%track.root_name\n zp_col = '%s_Zposition'%track.root_name\n \n #rot_order = track.skeleton[track.root_name]['order']\n #%(joint, rot_order[0])\n\n rot_order = track.skeleton[track.root_name]['order']\n r1_col = '%s_%srotation'%(track.root_name, rot_order[0])\n r2_col = '%s_%srotation'%(track.root_name, rot_order[1])\n r3_col = '%s_%srotation'%(track.root_name, rot_order[2])\n\n # Delta columns\n # dxp_col = '%s_dXposition'%track.root_name\n # dzp_col = '%s_dZposition'%track.root_name\n\n # dxr_col = '%s_dXrotation'%track.root_name\n # dyr_col = '%s_dYrotation'%track.root_name\n # dzr_col = '%s_dZrotation'%track.root_name\n dxp_col = 'reference_dXposition'\n dyp_col = 'reference_dYposition'\n dzp_col = 'reference_dZposition'\n dxr_col = 'reference_dXrotation'\n dyr_col = 'reference_dYrotation'\n dzr_col = 'reference_dZrotation'\n\n positions = np.transpose(np.array([track.values[xp_col], track.values[yp_col], track.values[zp_col]]))\n rotations = np.pi/180.0*np.transpose(np.array([track.values[r1_col], track.values[r2_col], track.values[r3_col]]))\n \n \"\"\" Get Trajectory and smooth it\"\"\" \n trajectory_filterwidth = self.position_smoothing\n #reference = positions.copy()*np.array([1,0,1])\n if trajectory_filterwidth>0:\n reference = filters.gaussian_filter1d(positions, trajectory_filterwidth, axis=0, mode='nearest')\n \n \"\"\" Get Root Velocity \"\"\"\n velocity = np.diff(reference, axis=0) \n velocity = np.vstack((velocity[0,:], velocity))\n\n \"\"\" Remove Root Translation \"\"\"\n positions = positions-reference\n\n \"\"\" Get Forward Direction along the x-z plane, assuming character is facig z-forward \"\"\"\n #forward = [Rotation(f, 'euler', from_deg=True, order=rot_order).rotmat[:,2] for f in rotations] # get the z-axis of the rotation matrix, assuming character is facig z-forward\n #print(\"order:\" + rot_order.lower())\n quats = Quaternions.from_euler(rotations, order=rot_order.lower(), world=False)\n\n #calculate the hips forward directions given in global cordinates \n #side_ax = np.zeros((1,3))\n #side_ax[0,self.hips_side_axis]=1\n #side_dirs = quats*side_ax\n side_dirs = quats*self.hips_side_axis\n forward = np.cross(np.array([[0,1,0]]), side_dirs)\n\n \"\"\" Smooth Forward Direction \"\"\" \n direction_filterwidth = self.rotation_smoothing\n if direction_filterwidth>0:\n forward = filters.gaussian_filter1d(forward, direction_filterwidth, axis=0, mode='nearest') \n\n # make unit vector\n forward = forward / np.sqrt((forward**2).sum(axis=-1))[...,np.newaxis]\n\n \"\"\" Remove Y Rotation \"\"\"\n target = np.array([[0,0,1]]).repeat(len(forward), axis=0)\n rotation = Quaternions.between(target, forward)[:,np.newaxis] \n positions = (-rotation[:,0]) * positions\n new_rotations = (-self.root_rotation_offset) * (-rotation[:,0]) * quats\n\n \"\"\" Get Root Rotation \"\"\"\n #print(rotation[:,0])\n velocity = (-rotation[:,0]) * velocity\n rvelocity = Pivots.from_quaternions(rotation[1:] * -rotation[:-1]).ps\n rvelocity = np.vstack((rvelocity[0], rvelocity))\n\n eulers = np.array([t3d.euler.quat2euler(q, axes=('s'+rot_order.lower()[::-1]))[::-1] for q in new_rotations])*180.0/np.pi\n \n new_df = track.values.copy()\n\n root_pos_x = pd.Series(data=positions[:,0], index=new_df.index)\n root_pos_y = pd.Series(data=positions[:,1], index=new_df.index)\n root_pos_z = pd.Series(data=positions[:,2], index=new_df.index)\n root_pos_x_diff = pd.Series(data=velocity[:,0], index=new_df.index)\n root_pos_y_diff = pd.Series(data=velocity[:,1], index=new_df.index)\n root_pos_z_diff = pd.Series(data=velocity[:,2], index=new_df.index)\n\n root_rot_1 = pd.Series(data=eulers[:,0], index=new_df.index)\n root_rot_2 = pd.Series(data=eulers[:,1], index=new_df.index)\n root_rot_3 = pd.Series(data=eulers[:,2], index=new_df.index)\n root_rot_y_diff = pd.Series(data=rvelocity[:,0], index=new_df.index)\n \n #new_df.drop([xr_col, yr_col, zr_col, xp_col, zp_col], axis=1, inplace=True)\n\n new_df[xp_col] = root_pos_x\n new_df[yp_col] = root_pos_y\n new_df[zp_col] = root_pos_z\n new_df[dxp_col] = root_pos_x_diff\n new_df[dyp_col] = root_pos_y_diff\n new_df[dzp_col] = root_pos_z_diff\n\n new_df[r1_col] = root_rot_1\n new_df[r2_col] = root_rot_2\n new_df[r3_col] = root_rot_3\n #new_df[dxr_col] = root_rot_x_diff\n new_df[dyr_col] = root_rot_y_diff\n #new_df[dzr_col] = root_rot_z_diff\n\n new_track.values = new_df\n\n\n elif self.method == 'hip_centric':\n new_track = track.clone()\n\n # Absolute columns\n xp_col = '%s_Xposition'%track.root_name\n yp_col = '%s_Yposition'%track.root_name\n zp_col = '%s_Zposition'%track.root_name\n\n xr_col = '%s_Xrotation'%track.root_name\n yr_col = '%s_Yrotation'%track.root_name\n zr_col = '%s_Zrotation'%track.root_name\n \n new_df = track.values.copy()\n\n all_zeros = np.zeros(track.values[xp_col].values.shape)\n\n new_df[xp_col] = pd.Series(data=all_zeros, index=new_df.index)\n new_df[yp_col] = pd.Series(data=all_zeros, index=new_df.index)\n new_df[zp_col] = pd.Series(data=all_zeros, index=new_df.index)\n new_df[zp_col] = pd.Series(data=all_zeros, index=new_df.index)\n\n new_df[xr_col] = pd.Series(data=all_zeros, index=new_df.index)\n new_df[yr_col] = pd.Series(data=all_zeros, index=new_df.index)\n new_df[zr_col] = pd.Series(data=all_zeros, index=new_df.index)\n\n new_track.values = new_df\n\n #print(new_track.values.columns)\n Q.append(new_track)\n\n return Q\n\n def inverse_transform(self, X, copy=None, start_pos=None):\n Q = []\n\n #TODO: simplify this implementation\n\n startx = 0\n startz = 0\n\n if start_pos is not None:\n startx, startz = start_pos\n\n for track in X:\n new_track = track.clone()\n if self.method == 'abdolute_translation_deltas':\n new_df = new_track.values\n xpcol = '%s_Xposition'%track.root_name\n ypcol = '%s_Yposition'%track.root_name\n zpcol = '%s_Zposition'%track.root_name\n\n\n dxpcol = '%s_dXposition'%track.root_name\n dzpcol = '%s_dZposition'%track.root_name\n\n dx = track.values[dxpcol].values\n dz = track.values[dzpcol].values\n\n recx = [startx]\n recz = [startz]\n\n for i in range(dx.shape[0]-1):\n recx.append(recx[i]+dx[i+1])\n recz.append(recz[i]+dz[i+1])\n\n # recx = [recx[i]+dx[i+1] for i in range(dx.shape[0]-1)]\n # recz = [recz[i]+dz[i+1] for i in range(dz.shape[0]-1)]\n # recx = dx[:-1] + dx[1:]\n # recz = dz[:-1] + dz[1:]\n if self.position_smoothing > 0: \n new_df[xpcol] = pd.Series(data=new_df[xpcol]+recx, index=new_df.index)\n new_df[zpcol] = pd.Series(data=new_df[zpcol]+recz, index=new_df.index)\n else:\n new_df[xpcol] = pd.Series(data=recx, index=new_df.index)\n new_df[zpcol] = pd.Series(data=recz, index=new_df.index)\n\n new_df.drop([dxpcol, dzpcol], axis=1, inplace=True)\n \n new_track.values = new_df\n # end of abdolute_translation_deltas\n \n elif self.method == 'pos_rot_deltas':\n # Absolute columns\n rot_order = track.skeleton[track.root_name]['order']\n xp_col = '%s_Xposition'%track.root_name\n yp_col = '%s_Yposition'%track.root_name\n zp_col = '%s_Zposition'%track.root_name\n\n xr_col = '%s_Xrotation'%track.root_name\n yr_col = '%s_Yrotation'%track.root_name\n zr_col = '%s_Zrotation'%track.root_name\n r1_col = '%s_%srotation'%(track.root_name, rot_order[0])\n r2_col = '%s_%srotation'%(track.root_name, rot_order[1])\n r3_col = '%s_%srotation'%(track.root_name, rot_order[2])\n\n # Delta columns\n # dxp_col = '%s_dXposition'%track.root_name\n # dzp_col = '%s_dZposition'%track.root_name\n # dyr_col = '%s_dYrotation'%track.root_name\n dxp_col = 'reference_dXposition'\n dzp_col = 'reference_dZposition'\n dyr_col = 'reference_dYrotation'\n\n positions = np.transpose(np.array([track.values[xp_col], track.values[yp_col], track.values[zp_col]]))\n rotations = np.pi/180.0*np.transpose(np.array([track.values[r1_col], track.values[r2_col], track.values[r3_col]]))\n quats = Quaternions.from_euler(rotations, order=rot_order.lower(), world=False)\n\n new_df = track.values.copy()\n\n dx = track.values[dxp_col].values\n dz = track.values[dzp_col].values\n\n dry = track.values[dyr_col].values\n\n #rec_p = np.array([startx, 0, startz])+positions[0,:]\n rec_ry = Quaternions.id(quats.shape[0])\n rec_xp = [0]\n rec_zp = [0]\n\n #rec_r = Quaternions.id(quats.shape[0])\n\n for i in range(dx.shape[0]-1):\n #print(dry[i])\n q_y = Quaternions.from_angle_axis(np.array(dry[i+1]), np.array([0,1,0]))\n rec_ry[i+1] = q_y*rec_ry[i]\n #print(\"dx: + \" + str(dx[i+1]))\n dp = rec_ry[i+1]*np.array([dx[i+1], 0, dz[i+1]])\n rec_xp.append(rec_xp[i]+dp[0,0])\n rec_zp.append(rec_zp[i]+dp[0,2])\n \n if self.separate_root:\n qq = quats\n xx = positions[:,0]\n zz = positions[:,2]\n else:\n qq = rec_ry*self.root_rotation_offset*quats\n pp = rec_ry*positions\n xx = rec_xp + pp[:,0]\n zz = rec_zp + pp[:,2]\n \n eulers = np.array([t3d.euler.quat2euler(q, axes=('s'+rot_order.lower()[::-1]))[::-1] for q in qq])*180.0/np.pi\n \n new_df = track.values.copy()\n\n root_rot_1 = pd.Series(data=eulers[:,0], index=new_df.index)\n root_rot_2 = pd.Series(data=eulers[:,1], index=new_df.index)\n root_rot_3 = pd.Series(data=eulers[:,2], index=new_df.index)\n \n new_df[xp_col] = pd.Series(data=xx, index=new_df.index)\n new_df[zp_col] = pd.Series(data=zz, index=new_df.index)\n\n new_df[r1_col] = pd.Series(data=root_rot_1, index=new_df.index)\n new_df[r2_col] = pd.Series(data=root_rot_2, index=new_df.index)\n new_df[r3_col] = pd.Series(data=root_rot_3, index=new_df.index)\n\n if self.separate_root:\n ref_rot_order=\"ZXY\"\n new_df[\"reference_Xposition\"] = pd.Series(data=rec_xp, index=new_df.index)\n new_df[\"reference_Zposition\"] = pd.Series(data=rec_zp, index=new_df.index) \n eulers_ry = np.array([t3d.euler.quat2euler(q, axes=('s'+ref_rot_order.lower()[::-1]))[::-1] for q in rec_ry])*180.0/np.pi\n new_df[\"reference_Yrotation\"] = pd.Series(data=eulers_ry[:,ref_rot_order.find('Y')], index=new_df.index)\n \n\n new_df.drop([dyr_col, dxp_col, dzp_col], axis=1, inplace=True)\n\n\n new_track.values = new_df\n \n elif self.method == 'pos_xyz_rot_deltas':\n # Absolute columns\n rot_order = track.skeleton[track.root_name]['order']\n xp_col = '%s_Xposition'%track.root_name\n yp_col = '%s_Yposition'%track.root_name\n zp_col = '%s_Zposition'%track.root_name\n\n xr_col = '%s_Xrotation'%track.root_name\n yr_col = '%s_Yrotation'%track.root_name\n zr_col = '%s_Zrotation'%track.root_name\n r1_col = '%s_%srotation'%(track.root_name, rot_order[0])\n r2_col = '%s_%srotation'%(track.root_name, rot_order[1])\n r3_col = '%s_%srotation'%(track.root_name, rot_order[2])\n\n # Delta columns\n # dxp_col = '%s_dXposition'%track.root_name\n # dzp_col = '%s_dZposition'%track.root_name\n # dyr_col = '%s_dYrotation'%track.root_name\n dxp_col = 'reference_dXposition'\n dyp_col = 'reference_dYposition'\n dzp_col = 'reference_dZposition'\n dyr_col = 'reference_dYrotation'\n\n positions = np.transpose(np.array([track.values[xp_col], track.values[yp_col], track.values[zp_col]]))\n rotations = np.pi/180.0*np.transpose(np.array([track.values[r1_col], track.values[r2_col], track.values[r3_col]]))\n quats = Quaternions.from_euler(rotations, order=rot_order.lower(), world=False)\n\n new_df = track.values.copy()\n\n dx = track.values[dxp_col].values\n dy = track.values[dyp_col].values\n dz = track.values[dzp_col].values\n\n dry = track.values[dyr_col].values\n\n #rec_p = np.array([startx, 0, startz])+positions[0,:]\n rec_ry = Quaternions.id(quats.shape[0])\n rec_xp = [0]\n rec_yp = [0]\n rec_zp = [0]\n\n #rec_r = Quaternions.id(quats.shape[0])\n\n for i in range(dx.shape[0]-1):\n #print(dry[i])\n q_y = Quaternions.from_angle_axis(np.array(dry[i+1]), np.array([0,1,0]))\n rec_ry[i+1] = q_y*rec_ry[i]\n #print(\"dx: + \" + str(dx[i+1]))\n dp = rec_ry[i+1]*np.array([dx[i+1], dy[i+1], dz[i+1]])\n rec_xp.append(rec_xp[i]+dp[0,0])\n rec_yp.append(rec_yp[i]+dp[0,1])\n rec_zp.append(rec_zp[i]+dp[0,2])\n \n if self.separate_root:\n qq = quats\n xx = positions[:,0]\n yy = positions[:,1]\n zz = positions[:,2]\n else:\n qq = rec_ry*self.root_rotation_offset*quats\n pp = rec_ry*positions\n xx = rec_xp + pp[:,0]\n yy = rec_yp + pp[:,1]\n zz = rec_zp + pp[:,2]\n \n eulers = np.array([t3d.euler.quat2euler(q, axes=('s'+rot_order.lower()[::-1]))[::-1] for q in qq])*180.0/np.pi\n \n new_df = track.values.copy()\n\n root_rot_1 = pd.Series(data=eulers[:,0], index=new_df.index)\n root_rot_2 = pd.Series(data=eulers[:,1], index=new_df.index)\n root_rot_3 = pd.Series(data=eulers[:,2], index=new_df.index)\n \n new_df[xp_col] = pd.Series(data=xx, index=new_df.index)\n new_df[yp_col] = pd.Series(data=yy, index=new_df.index)\n new_df[zp_col] = pd.Series(data=zz, index=new_df.index)\n\n new_df[r1_col] = pd.Series(data=root_rot_1, index=new_df.index)\n new_df[r2_col] = pd.Series(data=root_rot_2, index=new_df.index)\n new_df[r3_col] = pd.Series(data=root_rot_3, index=new_df.index)\n\n if self.separate_root:\n new_df[\"reference_Xposition\"] = pd.Series(data=rec_xp, index=new_df.index)\n new_df[\"reference_Yposition\"] = pd.Series(data=rec_yp, index=new_df.index)\n new_df[\"reference_Zposition\"] = pd.Series(data=rec_zp, index=new_df.index) \n eulers_ry = np.array([t3d.euler.quat2euler(q, axes=('s'+rot_order.lower()[::-1]))[::-1] for q in rec_ry])*180.0/np.pi\n new_df[\"reference_Yrotation\"] = pd.Series(data=eulers_ry[:,rot_order.find('Y')], index=new_df.index)\n \n\n new_df.drop([dyr_col, dxp_col, dyp_col, dzp_col], axis=1, inplace=True)\n\n\n new_track.values = new_df\n \n #print(new_track.values.columns)\n Q.append(new_track)\n\n return Q\n\n\n\nclass RootCentricPositionNormalizer(BaseEstimator, TransformerMixin):\n def __init__(self):\n pass\n\n def fit(self, X, y=None):\n return self\n\n def transform(self, X, y=None):\n Q = []\n\n for track in X:\n new_track = track.clone()\n\n rxp = '%s_Xposition'%track.root_name\n ryp = '%s_Yposition'%track.root_name\n rzp = '%s_Zposition'%track.root_name\n\n projected_root_pos = track.values[[rxp, ryp, rzp]]\n\n projected_root_pos.loc[:,ryp] = 0 # we want the root's projection on the floor plane as the ref\n\n new_df = pd.DataFrame(index=track.values.index)\n\n all_but_root = [joint for joint in track.skeleton if track.root_name not in joint]\n # all_but_root = [joint for joint in track.skeleton]\n for joint in all_but_root: \n new_df['%s_Xposition'%joint] = pd.Series(data=track.values['%s_Xposition'%joint]-projected_root_pos[rxp], index=new_df.index)\n new_df['%s_Yposition'%joint] = pd.Series(data=track.values['%s_Yposition'%joint]-projected_root_pos[ryp], index=new_df.index)\n new_df['%s_Zposition'%joint] = pd.Series(data=track.values['%s_Zposition'%joint]-projected_root_pos[rzp], index=new_df.index)\n \n\n # keep the root as it is now\n new_df[rxp] = track.values[rxp]\n new_df[ryp] = track.values[ryp]\n new_df[rzp] = track.values[rzp]\n\n new_track.values = new_df\n\n Q.append(new_track)\n \n return Q\n\n def inverse_transform(self, X, copy=None):\n Q = []\n\n for track in X:\n new_track = track.clone()\n\n rxp = '%s_Xposition'%track.root_name\n ryp = '%s_Yposition'%track.root_name\n rzp = '%s_Zposition'%track.root_name\n\n projected_root_pos = track.values[[rxp, ryp, rzp]]\n\n projected_root_pos.loc[:,ryp] = 0 # we want the root's projection on the floor plane as the ref\n\n new_df = pd.DataFrame(index=track.values.index)\n\n for joint in track.skeleton: \n new_df['%s_Xposition'%joint] = pd.Series(data=track.values['%s_Xposition'%joint]+projected_root_pos[rxp], index=new_df.index)\n new_df['%s_Yposition'%joint] = pd.Series(data=track.values['%s_Yposition'%joint]+projected_root_pos[ryp], index=new_df.index)\n new_df['%s_Zposition'%joint] = pd.Series(data=track.values['%s_Zposition'%joint]+projected_root_pos[rzp], index=new_df.index)\n \n\n new_track.values = new_df\n\n Q.append(new_track)\n \n return Q\n\nclass Flattener(BaseEstimator, TransformerMixin):\n def __init__(self):\n pass\n\n def fit(self, X, y=None):\n return self\n\n def transform(self, X, y=None):\n return np.concatenate(X, axis=0)\n\nclass ConstantsRemover(BaseEstimator, TransformerMixin):\n '''\n For now it just looks at the first track\n '''\n\n def __init__(self, eps = 1e-6):\n self.eps = eps\n \n\n def fit(self, X, y=None):\n stds = X[0].values.std()\n cols = X[0].values.columns.values\n self.const_dims_ = [c for c in cols if (stds[c] < self.eps).any()]\n self.const_values_ = {c:X[0].values[c].values[0] for c in cols if (stds[c] < self.eps).any()}\n return self\n\n def transform(self, X, y=None):\n Q = []\n \n\n for track in X:\n t2 = track.clone()\n #for key in t2.skeleton.keys():\n # if key in self.ConstDims_:\n # t2.skeleton.pop(key)\n #print(track.values.columns.difference(self.const_dims_))\n t2.values.drop(self.const_dims_, axis=1, inplace=True)\n #t2.values = track.values[track.values.columns.difference(self.const_dims_)]\n Q.append(t2)\n \n return Q\n \n def inverse_transform(self, X, copy=None):\n Q = []\n \n for track in X:\n t2 = track.clone()\n for d in self.const_dims_:\n t2.values[d] = self.const_values_[d]\n# t2.values.assign(d=pd.Series(data=self.const_values_[d], index = t2.values.index))\n Q.append(t2)\n\n return Q\n\nclass ListStandardScaler(BaseEstimator, TransformerMixin):\n def __init__(self, is_DataFrame=False):\n self.is_DataFrame = is_DataFrame\n \n def fit(self, X, y=None):\n if self.is_DataFrame:\n X_train_flat = np.concatenate([m.values for m in X], axis=0)\n else:\n X_train_flat = np.concatenate([m for m in X], axis=0)\n\n self.data_mean_ = np.mean(X_train_flat, axis=0)\n self.data_std_ = np.std(X_train_flat, axis=0)\n\n return self\n \n def transform(self, X, y=None):\n Q = []\n \n for track in X:\n if self.is_DataFrame:\n normalized_track = track.copy()\n normalized_track.values = (track.values - self.data_mean_) / self.data_std_\n else:\n normalized_track = (track - self.data_mean_) / self.data_std_\n\n Q.append(normalized_track)\n \n if self.is_DataFrame:\n return Q\n else:\n return np.array(Q)\n\n def inverse_transform(self, X, copy=None):\n Q = []\n \n for track in X:\n \n if self.is_DataFrame:\n unnormalized_track = track.copy()\n unnormalized_track.values = (track.values * self.data_std_) + self.data_mean_\n else:\n unnormalized_track = (track * self.data_std_) + self.data_mean_\n\n Q.append(unnormalized_track)\n \n if self.is_DataFrame:\n return Q\n else:\n return np.array(Q)\n\nclass ListMinMaxScaler(BaseEstimator, TransformerMixin):\n def __init__(self, is_DataFrame=False):\n self.is_DataFrame = is_DataFrame\n \n def fit(self, X, y=None):\n if self.is_DataFrame:\n X_train_flat = np.concatenate([m.values for m in X], axis=0)\n else:\n X_train_flat = np.concatenate([m for m in X], axis=0)\n\n self.data_max_ = np.max(X_train_flat, axis=0)\n self.data_min_ = np.min(X_train_flat, axis=0)\n\n return self\n \n def transform(self, X, y=None):\n Q = []\n \n for track in X:\n if self.is_DataFrame:\n normalized_track = track.copy()\n normalized_track.values = (track.values - self.data_min_) / (self.data_max_ - self.data_min_) \n else:\n normalized_track = (track - self.data_min_) / (self.data_max_ - self.data_min_)\n\n Q.append(normalized_track)\n \n if self.is_DataFrame:\n return Q\n else:\n return np.array(Q)\n\n def inverse_transform(self, X, copy=None):\n Q = []\n \n for track in X:\n \n if self.is_DataFrame:\n unnormalized_track = track.copy()\n unnormalized_track.values = (track.values * (self.data_max_ - self.data_min_)) + self.data_min_\n else:\n unnormalized_track = (track * (self.data_max_ - self.data_min_)) + self.data_min_\n\n Q.append(unnormalized_track)\n \n if self.is_DataFrame:\n return Q\n else:\n return np.array(Q)\n \nclass Resampler(BaseEstimator, TransformerMixin):\n def __init__(self, fps, method='cubic'):\n '''\n Method to resample a pandas dataframe to a different framerate.\n NOTE: Pandas resampling is quit unintuitive when resampling to odd framerates using interpolation.\n Thus we do it in this complex way.\n '''\n self.tgt_frametime = 1.0/fps\n self.method = method\n \n def fit(self, X, y=None):\n #print(\"Resampling to tgt_frametime: \" + str(self.tgt_frametime))\n self.orig_frametime=X[0].framerate\n return self\n \n def resample_dataframe(self, df, frametime, method='cubic'):\n #Create a time index for the resampled data\n rate = str(round(1.0e9*frametime))+'N'\n time_index = df.resample(rate).indices\n \n #reindex the old data. This will turn all non-matching indices to NAN\n tmp = df.reindex(time_index)\n \n #merge with the old data and sort\n tmp = pd.concat([df, tmp]).sort_index()\n \n #remove duplicate time indices. Then fill the NAN values using interpolation\n tmp=tmp[~tmp.index.duplicated(keep='first')].interpolate(method=method)\n\n #return the values using the resampled indices\n return tmp.loc[list(time_index)]\n \n def resample_df(self, df, new_frametime, old_frametime, mode='cubic'):\n\n #Create a time index for the resampled data\n data = df.values\n\n nframes = data.shape[0]\n nframes_new = round(nframes*old_frametime/new_frametime)\n x = np.arange(0, nframes)/(nframes-1)\n xnew = np.arange(0, nframes_new)/(nframes_new-1)\n\n data_out = np.zeros((nframes_new, data.shape[1]))\n for jj in range(data.shape[1]):\n y = data[:,jj]\n f = interpolate.interp1d(x, y, bounds_error=False, kind=mode, fill_value='extrapolate')\n data_out[:,jj] = f(xnew)\n\n time_index = pd.to_timedelta([f for f in range(xnew.shape[0])], unit='s')*new_frametime\n out = pd.DataFrame(data=data_out, index=time_index, columns=df.columns)\n \n #Scale root deltas to match new frame-rate\n sc = nframes/nframes_new\n rootdelta_cols = [c for c in df.columns if ('reference_d' in c)] \n out[rootdelta_cols]*=sc\n\n return out \n\n # def resample_poly_df(self, df, new_frametime, old_frametime):\n # old_fps = round(1/old_frametime)\n # new_fps = round(1/new_frametime)\n # lcm = np.lcm(old_fps, new_fps)\n # up = lcm//old_fps\n # down = lcm//new_fps\n # new_vals = signal.resample_poly(df.values, up, down, padtype='line')\n # time_index = pd.to_timedelta([f for f in range(new_vals.shape[0])], unit='s')*new_frametime\n # new_df = pd.DataFrame(data=new_vals, index=time_index, columns=df.columns)\n # return new_df\n \n def transform(self, X, y=None):\n Q = []\n \n for track in X:\n new_track = track.clone()\n # if self.method==\"resample_poly\":\n # new_track.values = self.resample_poly_df(track.values, self.tgt_frametime, track.framerate)\n # else:\n new_track.values = self.resample_df(track.values, self.tgt_frametime, track.framerate, self.method)\n #new_track.values = self.resample_dataframe(track.values, self.tgt_frametime, method=self.method)\n new_track.framerate = self.tgt_frametime\n Q.append(new_track)\n \n return Q\n \n def inverse_transform(self, X, copy=None):\n Q = []\n \n for track in X:\n new_track = track.clone()\n #new_track.values = self.resample_dataframe(track.values, self.orig_frametime, method=self.method)\n if self.method==\"resample_poly\":\n new_track.values = self.resample_poly_df(track.values, self.orig_frametime, track.framerate)\n else:\n new_track.values = self.resample_df(track.values, self.orig_frametime, track.framerate, self.method)\n new_track.framerate = self.orig_frametime\n Q.append(new_track)\n \n return Q\n \nclass DownSampler(BaseEstimator, TransformerMixin):\n def __init__(self, tgt_fps, keep_all=False):\n self.tgt_fps = tgt_fps\n self.keep_all = keep_all\n \n \n def fit(self, X, y=None): \n\n return self\n \n def transform(self, X, y=None):\n Q = []\n \n for track in X:\n orig_fps=round(1.0/track.framerate)\n rate = orig_fps//self.tgt_fps\n if orig_fps%self.tgt_fps!=0:\n print(\"error orig_fps (\" + str(orig_fps) + \") is not dividable with tgt_fps (\" + str(self.tgt_fps) + \")\")\n else:\n print(\"downsampling with rate: \" + str(rate))\n \n #print(track.values.size)\n for ii in range(0,rate):\n new_track = track.clone()\n if self.keep_all:\n new_track.take_name = new_track.take_name + \"_\" + str(ii).zfill(2)\n new_track.values = track.values[ii::rate].copy() \n #print(new_track.values.size)\n #new_track = track[0:-1:self.rate]\n new_track.framerate = 1.0/self.tgt_fps\n Q.append(new_track)\n if not self.keep_all:\n break\n \n return Q\n \n def inverse_transform(self, X, copy=None):\n return X\n\nclass ReverseTime(BaseEstimator, TransformerMixin):\n def __init__(self, append=True):\n self.append = append\n \n \n def fit(self, X, y=None): \n\n return self\n \n def transform(self, X, y=None):\n #print(\"ReverseTime\")\n Q = []\n if self.append:\n for track in X:\n Q.append(track)\n for track in X:\n new_track = track.clone() \n new_track.values = track.values[-1::-1]\n new_track.values.index=new_track.values.index[0]-new_track.values.index\n Q.append(new_track)\n \n return Q\n \n def inverse_transform(self, X, copy=None):\n return X\n\nclass ListFeatureUnion(BaseEstimator, TransformerMixin):\n def __init__(self, processors):\n self.processors = processors\n \n def fit(self, X, y=None):\n assert(y is None)\n for proc in self.processors:\n if isinstance(proc, Pipeline):\n #Loop steps and run fit on each. This is necessary since\n #running fit on a Pipeline runs fit_transform on all steps\n #and not only fit.\n for step in proc.steps:\n step[1].fit(X)\n else:\n proc.fit(X)\n return self\n \n def transform(self, X, y=None):\n\n assert(y is None)\n #print(\"ListFeatureUnion\")\n\n Q = []\n \n idx=0\n for proc in self.processors:\n Z = proc.transform(X)\n if idx==0:\n Q = Z\n else:\n assert(len(Q)==len(Z))\n for idx2,track in enumerate(Z):\n Q[idx2].values = pd.concat([Q[idx2].values,Z[idx2].values], axis=1)\n idx += 1\n \n return Q\n\n def inverse_transform(self, X, y=None):\n return X\n\nclass RollingStatsCalculator(BaseEstimator, TransformerMixin):\n '''\n Creates a causal mean and std filter with a rolling window of length win (based on using prev and current values)\n '''\n def __init__(self, win):\n self.win = win\n \n def fit(self, X, y=None): \n\n return self\n \n def transform(self, X, y=None):\n #print(\"RollingStatsCalculator: \" + str(self.win))\n\n Q = []\n for track in X:\n new_track = track.clone()\n mean_df = track.values.rolling(window=self.win).mean() \n std_df = track.values.rolling(window=self.win).std()\n # rolling.mean results in Nans in start seq. Here we fill these\n win = min(self.win, new_track.values.shape[0])\n for i in range(1,win):\n mm=track.values[:i].rolling(window=i).mean()\n ss=track.values[:i].rolling(window=i).std()\n mean_df.iloc[i-1] = mm.iloc[i-1]\n std_df.iloc[i-1] = ss.iloc[i-1]\n \n std_df.iloc[0] = std_df.iloc[1]\n # Append to\n new_track.values=pd.concat([mean_df.add_suffix('_mean'), std_df.add_suffix('_std')], axis=1)\n Q.append(new_track) \n return Q\n \n def inverse_transform(self, X, copy=None):\n return X\n\nclass FeatureCounter(BaseEstimator, TransformerMixin):\n def __init__(self):\n pass\n\n def fit(self, X, y=None):\n self.n_features = len(X[0].values.columns)\n\n return self\n\n def transform(self, X, y=None):\n return X\n\n def inverse_transform(self, X, copy=None):\n return X\n\n#TODO: JointsSelector (x)\n#TODO: SegmentMaker\n#TODO: DynamicFeaturesAdder\n#TODO: ShapeFeaturesAdder\n#TODO: DataFrameNumpier (x)\n\nclass TemplateTransform(BaseEstimator, TransformerMixin):\n def __init__(self):\n pass\n\n def fit(self, X, y=None):\n return self\n\n def transform(self, X, y=None):\n return X\n\n" }, { "path": "pymo/rotation_tools.py", "content": "'''\nTools for Manipulating and Converting 3D Rotations\n\nBy Omid Alemi\nCreated: June 12, 2017\n\nAdapted from that matlab file...\n'''\n\nimport math\nimport numpy as np\nimport transforms3d as t3d\nfrom pymo.Quaternions import Quaternions\n\ndef deg2rad(x):\n return x/180*math.pi\n\n\ndef rad2deg(x):\n return x/math.pi*180\n\ndef unroll(rots):\n new_rot = rots.copy()\n ang0 = np.linalg.norm(rots, axis=1) + 1e-8\n idx = np.where(ang0>np.pi)[0]\n ax = rots/np.tile(ang0[:,None], (1,3))\n ang1=ang0-2*np.pi\n alt_rot = ax*np.tile(ang1[:,None], (1,3))\n new_rot[idx] = alt_rot[idx]\n return new_rot \n\ndef unroll_1(rots):\n\n new_rots = rots.copy()\n \n # Compute angles and alternative rotation angles\n angs = np.linalg.norm(rots, axis=1)\n alt_angs=2*np.pi-angs\n\n #find discontinuities\n d_angs = np.diff(angs, axis=0)\n d_angs2 = alt_angs[1:]-angs[:-1]\n \n swps = np.where(np.abs(d_angs2)i', rots[:-1,:], rots[1:,:])\n #ax = rots/np.tile(angs[:, None], (1,3))\n #d_ax = np.linalg.norm(np.diff(ax, axis=0), axis=1)\n alt_angs=2*np.pi-angs\n\n #find discontinuities\n d_angs = np.diff(angs, axis=0)\n d_angs2 = alt_angs[1:]-angs[:-1]\n \n # FIXME should check if dot product is <0 not norm d_ax\n swps = np.where((dotprod<-1))[0]\n #swps = np.where((np.abs(d_ax)>0.5))[0]\n #swps = np.where(np.abs(d_angs2) 1.0e-10:\n vector = rot / theta\n else:\n vector = np.array([1.,0.,0.])\n theta=0.0\n eul = t3d.euler.axangle2euler(vector, theta, 'r' + order.lower())\n if use_deg:\n return np.rad2deg(eul)\n else:\n return eul\n\n\ndef euler2vectors(rot, order='XYZ', use_deg=False):\n if use_deg:\n rot = np.deg2rad(rot)\n\n # order = \"r\" + (order.lower())[::-1] # try both r and s\n # rotation_matrix = np.transpose(t3d.euler.euler2mat(rot[0], rot[1], rot[2], axes=order))\n rotation_matrix = t3d.euler.euler2mat(rot[0], rot[1], rot[2], 'r' + order.lower())\n\n y_vector_x_value = rotation_matrix[0][1]\n y_vector_y_value = rotation_matrix[1][1]\n y_vector_z_value = rotation_matrix[2][1]\n z_vector_x_value = rotation_matrix[0][2]\n z_vector_y_value = rotation_matrix[1][2]\n z_vector_z_value = rotation_matrix[2][2]\n\n return y_vector_x_value, y_vector_y_value, y_vector_z_value, z_vector_x_value, z_vector_y_value, z_vector_z_value\n\ndef vectors2euler(axises, order='XYZ', use_deg=False):\n \n # create rotation matrix\n y_vector = [axises[0], axises[1], axises[2]]\n z_vector = [axises[3], axises[4], axises[5]]\n x_vector = np.cross(y_vector, z_vector)\n\n R = np.column_stack((x_vector, y_vector, z_vector))\n \n # orthogonalize vectors\n u,s,vt = np.linalg.svd(R);\n R = np.matmul(u, vt);\n \n # to euler\n eul = t3d.euler.mat2euler(R, 'r' + order.lower())\n\n if use_deg:\n return np.rad2deg(eul)\n else:\n return eul\n\n\nclass Rotation():\n def __init__(self,rot, param_type, **params):\n self.rotmat = []\n if param_type == 'euler':\n self._from_euler(rot[0],rot[1],rot[2], params)\n elif param_type == 'expmap':\n self._from_expmap(rot[0], rot[1], rot[2], params)\n\n def _from_euler(self, alpha, beta, gamma, params):\n '''Expecting degress'''\n \n if params['from_deg']==True:\n alpha = deg2rad(alpha)\n beta = deg2rad(beta)\n gamma = deg2rad(gamma)\n \n order = \"s\" + ((params['order']).lower())[::-1]\n# Quaternions.from_euler() \n self.rotmat = np.transpose(t3d.euler.euler2mat(gamma, beta , alpha, axes=order))\n\n# ca = math.cos(alpha)\n# cb = math.cos(beta)\n# cg = math.cos(gamma)\n# sa = math.sin(alpha)\n# sb = math.sin(beta)\n# sg = math.sin(gamma) \n#\n# Rx = np.asarray([[1, 0, 0], \n# [0, ca, sa], \n# [0, -sa, ca]\n# ])\n#\n# Ry = np.asarray([[cb, 0, -sb], \n# [0, 1, 0],\n# [sb, 0, cb]])\n#\n# Rz = np.asarray([[cg, sg, 0],\n# [-sg, cg, 0],\n# [0, 0, 1]])\n#\n# self.rotmat = np.eye(3)\n# \n# order = params['order']\n# for i in range(0,len(order)):\n# if order[i]=='X':\n# self.rotmat = np.matmul(Rx, self.rotmat)\n# elif order[i]=='Y':\n# self.rotmat = np.matmul(Ry, self.rotmat)\n# elif order[i]=='Z':\n# self.rotmat = np.matmul(Rz, self.rotmat)\n# else:\n# print('unknown rotation axis: ' + order[i])\n# \n# # self.rotmat = np.matmul(np.matmul(Rz, Ry), Rx)\n# print (\"------\" + \"TRUE\")\n# print (self.rotmat)\n \n def _from_expmap(self, alpha, beta, gamma, params):\n if (alpha == 0 and beta == 0 and gamma == 0):\n self.rotmat = np.eye(3)\n return\n\n #TODO: Check exp map params\n\n theta = np.linalg.norm([alpha, beta, gamma])\n\n expmap = [alpha, beta, gamma] / theta\n\n x = expmap[0]\n y = expmap[1]\n z = expmap[2]\n\n s = math.sin(theta/2)\n c = math.cos(theta/2)\n\n self.rotmat = np.asarray([\n [2*(x**2-1)*s**2+1, 2*x*y*s**2-2*z*c*s, 2*x*z*s**2+2*y*c*s],\n [2*x*y*s**2+2*z*c*s, 2*(y**2-1)*s**2+1, 2*y*z*s**2-2*x*c*s],\n [2*x*z*s**2-2*y*c*s, 2*y*z*s**2+2*x*c*s , 2*(z**2-1)*s**2+1]\n ])\n \n\n\n def get_euler_axis(self):\n R = self.rotmat\n theta = math.acos((self.rotmat.trace() - 1) / 2)\n axis = np.asarray([R[2,1] - R[1,2], R[0,2] - R[2,0], R[1,0] - R[0,1]])\n axis = axis/(2*math.sin(theta))\n return theta, axis\n\n def to_expmap(self):\n axis, theta = t3d.axangles.mat2axangle(self.rotmat, unit_thresh=1e-05)\n# theta, axis = self.get_euler_axis()\n rot_arr = theta * axis\n if np.isnan(rot_arr).any():\n rot_arr = [0, 0, 0]\n return rot_arr\n \n def to_euler(self, use_deg=False, order='xyz'):\n order = \"s\" + order.lower()\n eulers = t3d.euler.mat2euler(np.transpose(self.rotmat), axes=order)\n return eulers[::-1]\n \n# eulers = np.zeros((2, 3))\n#\n# if np.absolute(np.absolute(self.rotmat[2, 0]) - 1) < 1e-12:\n# #GIMBAL LOCK!\n# print('Gimbal')\n# if np.absolute(self.rotmat[2, 0]) - 1 < 1e-12:\n# eulers[:,0] = math.atan2(-self.rotmat[0,1], -self.rotmat[0,2])\n# eulers[:,1] = -math.pi/2\n# else:\n# eulers[:,0] = math.atan2(self.rotmat[0,1], -elf.rotmat[0,2])\n# eulers[:,1] = math.pi/2\n# \n# return eulers\n#\n# theta = - math.asin(self.rotmat[2,0])\n# theta2 = math.pi - theta\n#\n# # psi1, psi2\n# eulers[0,0] = math.atan2(self.rotmat[2,1]/math.cos(theta), self.rotmat[2,2]/math.cos(theta))\n# eulers[1,0] = math.atan2(self.rotmat[2,1]/math.cos(theta2), self.rotmat[2,2]/math.cos(theta2))\n#\n# # theta1, theta2\n# eulers[0,1] = theta\n# eulers[1,1] = theta2\n#\n# # phi1, phi2\n# eulers[0,2] = math.atan2(self.rotmat[1,0]/math.cos(theta), self.rotmat[0,0]/math.cos(theta))\n# eulers[1,2] = math.atan2(self.rotmat[1,0]/math.cos(theta2), self.rotmat[0,0]/math.cos(theta2))\n#\n if use_deg:\n eulers = rad2deg(eulers)\n\n return eulers\n \n def to_quat(self):\n #TODO\n pass\n \n def __str__(self):\n return \"Rotation Matrix: \\n \" + self.rotmat.__str__()\n \n\n\n\n" }, { "path": "pymo/viz_tools.py", "content": "import pandas as pd\nimport numpy as np\nimport matplotlib.animation as animation\nimport matplotlib.colors as colors\nimport matplotlib.patheffects as pe\nimport matplotlib.pyplot as plt\n#import IPython\nimport os\n\ndef save_fig(fig_id, tight_layout=True):\n if tight_layout:\n plt.tight_layout()\n plt.savefig(fig_id + '.png', format='png', dpi=300)\n \n \ndef draw_stickfigure(mocap_track, frame, data=None, joints=None, draw_names=False, ax=None, figsize=(8,8)):\n if ax is None:\n fig = plt.figure(figsize=figsize)\n ax = fig.add_subplot(111)\n \n if joints is None:\n joints_to_draw = mocap_track.skeleton.keys()\n else:\n joints_to_draw = joints\n \n if data is None:\n df = mocap_track.values\n else:\n df = data\n \n for joint in joints_to_draw:\n ax.scatter(x=df['%s_Xposition'%joint][frame], \n y=df['%s_Yposition'%joint][frame], \n alpha=0.6, c='b', marker='o')\n\n parent_x = df['%s_Xposition'%joint][frame]\n parent_y = df['%s_Yposition'%joint][frame]\n \n children_to_draw = [c for c in mocap_track.skeleton[joint]['children'] if c in joints_to_draw]\n \n for c in children_to_draw:\n child_x = df['%s_Xposition'%c][frame]\n child_y = df['%s_Yposition'%c][frame]\n ax.plot([parent_x, child_x], [parent_y, child_y], 'k-', lw=2)\n \n if draw_names:\n ax.annotate(joint, \n (df['%s_Xposition'%joint][frame] + 0.1, \n df['%s_Yposition'%joint][frame] + 0.1))\n\n return ax\n\ndef draw_stickfigure3d(mocap_track, frame, data=None, joints=None, draw_names=False, ax=None, figsize=(8,8)):\n from mpl_toolkits.mplot3d import Axes3D\n \n if ax is None:\n fig = plt.figure(figsize=figsize)\n ax = fig.add_subplot(111, projection='3d') \n \n if joints is None:\n joints_to_draw = mocap_track.skeleton.keys()\n else:\n joints_to_draw = joints\n \n if data is None:\n df = mocap_track.values\n else:\n df = data\n \n for joint in joints_to_draw:\n parent_x = df['%s_Xposition'%joint][frame]\n parent_y = df['%s_Zposition'%joint][frame]\n parent_z = df['%s_Yposition'%joint][frame]\n # ^ In mocaps, Y is the up-right axis \n\n ax.scatter(xs=parent_x, \n ys=parent_y, \n zs=parent_z, \n alpha=0.6, c='b', marker='o')\n\n \n children_to_draw = [c for c in mocap_track.skeleton[joint]['children'] if c in joints_to_draw]\n \n for c in children_to_draw:\n child_x = df['%s_Xposition'%c][frame]\n child_y = df['%s_Zposition'%c][frame]\n child_z = df['%s_Yposition'%c][frame]\n # ^ In mocaps, Y is the up-right axis\n\n ax.plot([parent_x, child_x], [parent_y, child_y], [parent_z, child_z], 'k-', lw=2, c='black')\n \n if draw_names:\n ax.text(x=parent_x + 0.1, \n y=parent_y + 0.1,\n z=parent_z + 0.1,\n s=joint,\n color='rgba(0,0,0,0.9')\n\n return ax\n\n\ndef sketch_move(mocap_track, data=None, ax=None, figsize=(16,8)):\n if ax is None:\n fig = plt.figure(figsize=figsize)\n ax = fig.add_subplot(111)\n \n if data is None:\n data = mocap_track.values\n\n for frame in range(0, data.shape[0], 4):\n# draw_stickfigure(mocap_track, f, data=data, ax=ax)\n \n for joint in mocap_track.skeleton.keys():\n children_to_draw = [c for c in mocap_track.skeleton[joint]['children']]\n \n parent_x = data['%s_Xposition'%joint][frame]\n parent_y = data['%s_Yposition'%joint][frame]\n \n frame_alpha = frame/data.shape[0]\n \n for c in children_to_draw:\n child_x = data['%s_Xposition'%c][frame]\n child_y = data['%s_Yposition'%c][frame]\n \n ax.plot([parent_x, child_x], [parent_y, child_y], '-', lw=1, color='gray', alpha=frame_alpha)\n\ndef render_mp4(mocap_track, filename, data=None, ax=None, axis_scale=50, elev=45, azim=45):\n if ax is None:\n fig = plt.figure(figsize=(10,10))\n ax = fig.add_subplot(111, projection='3d')\n ax.set_xlim3d(-axis_scale, axis_scale)\n ax.set_zlim3d( 0, axis_scale)\n ax.set_ylim3d(-axis_scale, axis_scale)\n ax.grid(True)\n ax.set_axis_off()\n\n ax.view_init(elev=elev, azim=azim)\n\n xs = np.linspace(-200, 200, 50)\n ys = np.linspace(-200, 200, 50)\n X, Y = np.meshgrid(xs, ys)\n Z = np.zeros(X.shape)\n \n wframe = ax.plot_wireframe(X, Y, Z, rstride=2, cstride=2, color='grey',lw=0.2)\n\n # fig = plt.figure(figsize=figsize)\n # ax = fig.add_subplot(111)\n \n if data is None:\n data = mocap_track.values\n \n fps=int(np.round(1/mocap_track.framerate))\n lines=[]\n lines.append([plt.plot([0,0], [0,0], [0,0], color='red', \n lw=2, path_effects=[pe.Stroke(linewidth=3, foreground='black'), pe.Normal()])[0] for _ in range(len(mocap_track.skeleton.keys()))])\n \n def animate(frame):\n \n changed = []\n j=0\n for joint in mocap_track.skeleton.keys():\n children_to_draw = [c for c in mocap_track.skeleton[joint]['children']]\n \n parent_x = data['%s_Xposition'%joint][frame]\n parent_y = data['%s_Yposition'%joint][frame]\n parent_z = data['%s_Zposition'%joint][frame]\n \n #frame_alpha = frame/data.shape[0]\n \n for c in children_to_draw:\n child_x = data['%s_Xposition'%c][frame]\n child_y = data['%s_Yposition'%c][frame]\n child_z = data['%s_Zposition'%c][frame]\n \n lines[0][j].set_data(np.array([[child_x, parent_x],[-child_z,-parent_z]]))\n lines[0][j].set_3d_properties(np.array([ child_y,parent_y]))\n\n changed += lines\n j+=1\n \n return changed\n \n plt.tight_layout()\n \n ani = animation.FuncAnimation(fig, \n animate, np.arange(data.shape[0]), interval=1000/fps)\n\n if filename != None:\n ani.save(filename, fps=fps, bitrate=13934)\n ani.event_source.stop()\n del ani\n plt.close() \n try:\n plt.show()\n plt.save()\n except AttributeError as e:\n pass\n \n\ndef viz_cnn_filter(feature_to_viz, mocap_track, data, gap=25):\n fig = plt.figure(figsize=(16,4))\n ax = plt.subplot2grid((1,8),(0,0))\n ax.imshow(feature_to_viz.T, aspect='auto', interpolation='nearest')\n \n ax = plt.subplot2grid((1,8),(0,1), colspan=7)\n for frame in range(feature_to_viz.shape[0]):\n frame_alpha = 0.2#frame/data.shape[0] * 2 + 0.2\n\n for joint_i, joint in enumerate(mocap_track.skeleton.keys()):\n children_to_draw = [c for c in mocap_track.skeleton[joint]['children']]\n\n parent_x = data['%s_Xposition'%joint][frame] + frame * gap\n parent_y = data['%s_Yposition'%joint][frame] \n\n ax.scatter(x=parent_x, \n y=parent_y, \n alpha=0.6,\n cmap='RdBu',\n c=feature_to_viz[frame][joint_i] * 10000,\n marker='o',\n s = abs(feature_to_viz[frame][joint_i] * 10000))\n plt.axis('off')\n for c in children_to_draw:\n child_x = data['%s_Xposition'%c][frame] + frame * gap\n child_y = data['%s_Yposition'%c][frame] \n\n ax.plot([parent_x, child_x], [parent_y, child_y], '-', lw=1, color='gray', alpha=frame_alpha)\n\n \ndef print_skel(X):\n stack = [X.root_name]\n tab=0\n while stack:\n joint = stack.pop()\n tab = len(stack)\n print('%s- %s (%s)'%('| '*tab, joint, X.skeleton[joint]['parent']))\n for c in X.skeleton[joint]['children']:\n stack.append(c)\n\n\n# def nb_play_mocap_fromurl(mocap, mf, frame_time=1/30, scale=1, base_url='http://titan:8385'):\n # if mf == 'bvh':\n # bw = BVHWriter()\n # with open('test.bvh', 'w') as ofile:\n # bw.write(mocap, ofile)\n \n # filepath = '../notebooks/test.bvh'\n # elif mf == 'pos':\n # c = list(mocap.values.columns)\n\n # for cc in c:\n # if 'rotation' in cc:\n # c.remove(cc)\n # mocap.values.to_csv('test.csv', index=False, columns=c)\n \n # filepath = '../notebooks/test.csv'\n # else:\n # return\n \n # url = '%s/mocapplayer/player.html?data_url=%s&scale=%f&cz=200&order=xzyi&frame_time=%f'%(base_url, filepath, scale, frame_time)\n # iframe = ''\n # link = 'New Window'%url\n # return IPython.display.HTML(iframe+link)\n\n# def nb_play_mocap(mocap, mf, meta=None, frame_time=1/30, scale=1, camera_z=500, base_url=None):\n # data_template = 'var dataBuffer = `$$DATA$$`;'\n # data_template += 'var metadata = $$META$$;'\n # data_template += 'start(dataBuffer, metadata, $$CZ$$, $$SCALE$$, $$FRAMETIME$$);'\n # dir_path = os.path.dirname(os.path.realpath(__file__))\n\n\n # if base_url is None:\n # base_url = os.path.join(dir_path, 'mocapplayer/playBuffer.html')\n \n # # print(dir_path)\n\n # if mf == 'bvh':\n # pass\n # elif mf == 'pos':\n # cols = list(mocap.values.columns)\n # for c in cols:\n # if 'rotation' in c:\n # cols.remove(c)\n \n # data_csv = mocap.values.to_csv(index=False, columns=cols)\n\n # if meta is not None:\n # lines = [','.join(item) for item in meta.astype('str')]\n # meta_csv = '[' + ','.join('[%s]'%l for l in lines) +']' \n # else:\n # meta_csv = '[]'\n \n # data_assigned = data_template.replace('$$DATA$$', data_csv)\n # data_assigned = data_assigned.replace('$$META$$', meta_csv)\n # data_assigned = data_assigned.replace('$$CZ$$', str(camera_z))\n # data_assigned = data_assigned.replace('$$SCALE$$', str(scale))\n # data_assigned = data_assigned.replace('$$FRAMETIME$$', str(frame_time))\n\n # else:\n # return\n \n \n\n # with open(os.path.join(dir_path, 'mocapplayer/data.js'), 'w') as oFile:\n # oFile.write(data_assigned)\n\n # url = '%s?&cz=200&order=xzyi&frame_time=%f&scale=%f'%(base_url, frame_time, scale)\n # iframe = ''\n # link = 'New Window'%url\n # return IPython.display.HTML(iframe+link)\n " }, { "path": "pymo/writers.py", "content": "import numpy as np\nimport pandas as pd\n\nclass BVHWriter():\n def __init__(self):\n pass\n \n def write(self, X, ofile, framerate=-1, start=0, stop=-1):\n \n # Writing the skeleton info\n ofile.write('HIERARCHY\\n')\n \n self.motions_ = []\n self._printJoint(X, X.root_name, 0, ofile)\n\n if stop > 0:\n nframes = stop-start\n else:\n nframes = X.values.shape[0]\n stop = X.values.shape[0]\n\n # Writing the motion header\n ofile.write('MOTION\\n')\n ofile.write('Frames: %d\\n'%nframes)\n \n if framerate > 0:\n ofile.write('Frame Time: %f\\n'%float(1.0/framerate))\n else:\n ofile.write('Frame Time: %f\\n'%X.framerate)\n\n # Writing the data\n self.motions_ = np.asarray(self.motions_).T\n lines = [\" \".join(item) for item in self.motions_[start:stop].astype(str)]\n ofile.write(\"\".join(\"%s\\n\"%l for l in lines))\n\n def _printJoint(self, X, joint, tab, ofile):\n \n if X.skeleton[joint]['parent'] == None:\n ofile.write('ROOT %s\\n'%joint)\n elif len(X.skeleton[joint]['children']) > 0:\n ofile.write('%sJOINT %s\\n'%('\\t'*(tab), joint))\n else:\n ofile.write('%sEnd site\\n'%('\\t'*(tab)))\n\n ofile.write('%s{\\n'%('\\t'*(tab)))\n \n ofile.write('%sOFFSET %3.5f %3.5f %3.5f\\n'%('\\t'*(tab+1),\n X.skeleton[joint]['offsets'][0],\n X.skeleton[joint]['offsets'][1],\n X.skeleton[joint]['offsets'][2]))\n rot_order = X.skeleton[joint]['order']\n \n #print(\"rot_order = \" + rot_order)\n channels = X.skeleton[joint]['channels']\n rot = [c for c in channels if ('rotation' in c)]\n pos = [c for c in channels if ('position' in c)]\n \n n_channels = len(rot) +len(pos)\n ch_str = ''\n if n_channels > 0:\n for ci in range(len(pos)):\n cn = pos[ci]\n self.motions_.append(np.asarray(X.values['%s_%s'%(joint,cn)].values))\n ch_str = ch_str + ' ' + cn \n for ci in range(len(rot)):\n cn = '%srotation'%(rot_order[ci])\n self.motions_.append(np.asarray(X.values['%s_%s'%(joint,cn)].values))\n ch_str = ch_str + ' ' + cn \n if len(X.skeleton[joint]['children']) > 0:\n #ch_str = ''.join(' %s'*n_channels%tuple(channels))\n ofile.write('%sCHANNELS %d%s\\n' %('\\t'*(tab+1), n_channels, ch_str)) \n\n for c in X.skeleton[joint]['children']:\n self._printJoint(X, c, tab+1, ofile)\n\n ofile.write('%s}\\n'%('\\t'*(tab)))\n" }, { "path": "requirements.txt", "content": "numpy\nscipy\nlibrosa\ntorchmetrics\npytorch-lightning==1.8.3\npandas\nmatplotlib\nscikit-learn==0.24.2\nmpi4py\njoblib\nsmplx\ntransforms3d\nx-transformers\nmadmom\nftfy\naxial_positional_embedding\neinops\nentmax\nlightning-bolts\nloguru\nmadgrad\nmido\njsmin\noptuna\n" }, { "path": "run_docker.sh", "content": "docker run -it --rm --gpus '\"device=0,1,2,3,4,5,6,7\"' -v $PWD:/workspace/dockers --ipc=host -v=$HOME/data:/workspace/dockers/data simonal_diffusion /bin/sh -c 'cd dockers; bash'\n" }, { "path": "synthesize.py", "content": "# Copyright 2023 Motorica AB, Inc. All Rights Reserved.\nfrom os.path import join\nimport os, sys, getopt\nimport torch\nimport numpy as np\nimport pickle as pkl\nfrom pytorch_lightning import Trainer, seed_everything\nfrom utils.motion_dataset import styles2onehot, nans2zeros\nfrom models.LightningModel import LitLDA\n\ndef sample_mixmodels(models, batches, guidance_factors):\n # asserts that the models are compatible, \n # i.e. they have the same number of noise steps, pose dim and pose scalers\n assert len(guidance_factors)==(len(models)-1), \"n_guidance_factors should be eq to n_models-1\"\n noise_sched_0 = models[0].noise_schedule\n o_scaler_0 = models[0].hparams[\"Data\"][\"scalers\"][\"out_scaler\"]\n eps = 0.000001\n for i in range(1, len(models)):\n # models should have same noise schedule \n assert torch.all(torch.abs(models[i].noise_schedule - noise_sched_0) 0:\n noise = torch.randn_like(poses)\n sigma = ((1.0 - alpha_cum[n-1]) / (1.0 - alpha_cum[n]) * beta[n])**0.5\n poses += sigma * noise\n \n out_poses = models[-1].destandardizeOutput(poses)\n if not models[-1].unconditional:\n out_ctrl = models[-1].destandardizeInput(ctrl)\n anim_clip = torch.cat((out_poses, out_ctrl), dim=2).cpu().detach().numpy() \n else:\n anim_clip = out_poses.cpu().detach().numpy()\n return anim_clip\n\ndef do_synthesize(models, l_conds, g_conds, file_name, postfix, trim, dest_dir, guidance_factors, gpu, render_video, outfile):\n nframes = l_conds[-1].size(1)\n \n device = torch.device(gpu)\n batches = []\n for i in range(len(models)):\n models[i].to(device)\n models[i].eval()\n batch = l_conds[i].to(device) if len(l_conds[i])>0 else [], g_conds[i].to(device) if len(g_conds[i])>0 else [], None \n batches.append(batch)\n \n with torch.no_grad():\n clips = sample_mixmodels(models, batches, guidance_factors) \n models[-1].log_results(clips[:,trim:nframes-trim,:], outfile, \"\", logdir=dest_dir, render_video=render_video)\n\ndef nans2zeros(x):\n ii = np.where(np.isinf(x))\n x[ii]=0\n ii = np.where(np.isnan(x))\n x[ii]=0\n return x\n \ndef get_style_vector(styles_file, style_token, nbatch, nframes):\n all_styles = np.loadtxt(styles_file, dtype=str) \n styles_onehot = styles2onehot(all_styles, style_token)\n styles = styles_onehot.repeat(nbatch, nframes,1) \n\ndef get_cond(model, data_dir, input_file, style_token, length):\n # Load input features\n with open(join(data_dir, input_file), 'rb') as f:\n ctrl = pkl.load(f)\n ctrl = ctrl[startframe:]\n if endframe>0 and endframe0:\n file_name = file_name + \"_\" + log_prefix\n \n bvh_data = self.feats_to_bvh(pred_clips)\n nclips = len(bvh_data)\n framerate = np.rint(1/bvh_data[0].framerate)\n \n if self.hparams.Validation[\"max_render_clips\"]:\n nclips = min(nclips, self.hparams.Validation[\"max_render_clips\"])\n \n self.write_bvh(bvh_data[:nclips], log_dir=logdir, name_prefix=file_name)\n \n if render_video:\n pos_data = self.bvh_to_pos(bvh_data)\n \n if render_video:\n self.render_video(pos_data[:nclips], log_dir=logdir, name_prefix=file_name)\n \n \n def feats_to_bvh(self, pred_clips):\n #import pdb;pdb.set_trace()\n data_pipeline = jl.load(Path(self.hparams.dataset_root) / self.hparams.Data[\"datapipe_filename\"])\n n_feats = data_pipeline[\"cnt\"].n_features \n data_pipeline[\"root\"].separate_root=False\n \n print('inverse_transform...')\n bvh_data=data_pipeline.inverse_transform(pred_clips[:,:,:n_feats])\n return bvh_data\n \n def write_bvh(self, bvh_data, log_dir=\"\", name_prefix=\"\"):\n writer = BVHWriter()\n nclips = len(bvh_data)\n for i in range(nclips): \n if nclips>1:\n fname = f\"{log_dir}/{name_prefix}_{str(i).zfill(3)}.bvh\"\n else:\n fname = f\"{log_dir}/{name_prefix}.bvh\"\n print('writing:' + fname)\n with open(fname,'w') as f:\n writer.write(bvh_data[i], f)\n \n def bvh_to_pos(self, bvh_data): \n # convert to joint positions\n return MocapParameterizer('position').fit_transform(bvh_data)\n \n def render_video(self, pos_data, log_dir=\"\", name_prefix=\"\"):\n # write bvh and skeleton motion\n nclips = len(pos_data)\n for i in range(nclips): \n if nclips>1:\n fname = f\"{log_dir}/{name_prefix}_{str(i).zfill(3)}\"\n else:\n fname = f\"{log_dir}/{name_prefix}\"\n print('writing:' + fname + \".mp4\")\n render_mp4(pos_data[i], fname + \".mp4\", axis_scale=200)\n \n \n def log_jerk(self, x, log_prefix):\n\n deriv = x[:, 1:] - x[:, :-1]\n acc = deriv[:, 1:] - deriv[:, :-1]\n jerk = acc[:, 1:] - acc[:, :-1]\n self.log(f'{log_prefix}_jerk', torch.mean(torch.abs(jerk)), sync_dist=True)\n \n" }, { "path": "utils/motion_dataset.py", "content": "# Copyright 2023 Motorica AB, Inc. All Rights Reserved.\n\nimport os\nimport random\nimport argparse\nimport json\nimport torch\nimport torch.utils.data\nimport sys\nfrom pathlib import Path\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler\nimport pickle as pkl\nimport pandas as pd\nfrom scipy import interpolate\n\ndef concat_dataframes(x,y):\n # Assume data is synched on the start time\n if x.shape[0]0)\n for i in range(len(inds)):\n assert len(tokens) > inds[i], f\"{inds[i]} out of range in {basename}\"\n out+=tokens[inds[i]]\n if i0:\n in_feats = in_feats[trim_edges:-trim_edges]\n out_feats = out_feats[trim_edges:-trim_edges]\n \n n_frames=in_feats.shape[0]\n\n # global conditioning (from file naming convention)\n if \"styles_file\" in data_hparams:\n styles_file = Path(data_root) / data_hparams[\"styles_file\"]\n all_styles = np.loadtxt(styles_file, dtype=str).tolist() \n styles_oh = np.tile(styles2onehot(all_styles, parse_token(files[fi], data_hparams[\"style_index\"])),(n_frames,1))\n self.n_styles = len(all_styles)\n else:\n self.n_styles = 0\n\n #we create indexes for full length sequences here\n seglen=max_segment_length\n if n_frames >= seglen:\n idx_array = torch.arange(start_idx, start_idx + n_frames).unfold(\n 0, seglen, 1\n )\n data[\"input\"].append(in_feats)\n data[\"output\"].append(out_feats)\n if self.n_styles>0:\n data[\"styles\"].append(styles_oh)\n indexes.append(idx_array) \n start_idx += n_frames\n \n #flatten vertically and make into a torch tensor\n data[\"input\"]=torch.from_numpy(np.vstack(data[\"input\"])).float()\n data[\"output\"]=torch.from_numpy(np.vstack(data[\"output\"])).float()\n if self.n_styles>0:\n data[\"styles\"]=torch.from_numpy(np.vstack(data[\"styles\"])).float() \n print(f\"=== tot number of frames: {data['output'].shape[0]} =====\")\n\n self.data = data\n\n indexes=torch.cat(indexes, dim=0)\n self.indexes = indexes[torch.randperm(indexes.size(0))]\n \n def assert_not_const(self, data):\n eps = 1e-6\n assert((data.std(axis=0)0:\n style_scaler = StandardScaler()\n style_scaler.mean_=np.zeros(self.n_styles)\n style_scaler.scale_=np.ones(self.n_styles)\n else:\n style_scaler=None\n\n return {\"in_scaler\": in_scaler, \"style_scaler\": style_scaler,\"out_scaler\": out_scaler}\n\n def standardize(self, scalers): \n self.data[\"input\"] = torch.from_numpy(scalers[\"in_scaler\"].transform(self.data[\"input\"])).float()\n self.data[\"output\"] = torch.from_numpy(scalers[\"out_scaler\"].transform(self.data[\"output\"])).float()\n \n def timestretch(self, data, segment_length, factor, has_root_motion=False):\n if factor<1.0:\n #Truncate original samples and stretch\n return resample_data(data[:int(factor*segment_length)],segment_length, has_root_motion)\n elif factor>1.0:\n #Stretch original samples and trunkate\n return resample_data(data,int(factor*segment_length),has_root_motion)[:segment_length]\n else:\n # return original\n return data[:segment_length]\n \n def __getitem__(self, index):\n in_feats = self.data[\"input\"][self.indexes[index]]\n out_feats = self.data[\"output\"][self.indexes[index]]\n \n if self.timestretch_factor>0:\n #note that the sequences are longer than specified so we can resample faster speeds\n if torch.rand((1,))0:\n styles = self.data[\"styles\"][self.indexes[index]]\n styles = styles[:self.segment_length]\n else:\n styles = []\n\n return (in_feats, styles, out_feats)\n\n def __len__(self):\n return self.indexes.size(0)\n " } ]