Repository: CaliOpen/Caliopen Branch: master Commit: 7ae023582358 Files: 2425 Total size: 10.9 MB Directory structure: gitextract_it3wvh4t/ ├── .drone.yml ├── .git-crypt/ │ ├── .gitattributes │ └── keys/ │ └── default/ │ └── 0/ │ ├── 3A1172187EF7406FCBCECFA60E68C4F85D04D8FC.gpg │ ├── A08EA82ED095FCD8575AB3742DC613553716A878.gpg │ ├── E6711893CE02FDEEA5B1F79FF3D83146E5111526.gpg │ └── EA440246DFB690B5B1FB4032CE011C4615DB040A.gpg ├── .gitattributes ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── devtools/ │ ├── ES_queries/ │ │ └── Recipient_suggest.json │ ├── Vagrantfile │ ├── api-mock/ │ │ ├── README.md │ │ ├── all.fixture.js │ │ ├── authentications/ │ │ │ ├── data.json │ │ │ └── index.js │ │ ├── bin/ │ │ │ └── start │ │ ├── collection-middleware/ │ │ │ └── index.js │ │ ├── contacts/ │ │ │ ├── data.json │ │ │ └── index.js │ │ ├── devices/ │ │ │ ├── data.json │ │ │ └── index.js │ │ ├── discussions/ │ │ │ ├── data.json │ │ │ └── index.js │ │ ├── local_identities/ │ │ │ ├── data.json │ │ │ └── index.js │ │ ├── me/ │ │ │ ├── data.json │ │ │ └── index.js │ │ ├── messages/ │ │ │ ├── data.json │ │ │ └── index.js │ │ ├── notifications/ │ │ │ ├── data.json │ │ │ └── index.js │ │ ├── oauth-mock/ │ │ │ ├── data.json │ │ │ └── index.js │ │ ├── package.json │ │ ├── participants/ │ │ │ ├── data.json │ │ │ └── index.js │ │ ├── providers/ │ │ │ ├── data.json │ │ │ └── index.js │ │ ├── remote_identities/ │ │ │ ├── data.json │ │ │ └── index.js │ │ ├── search/ │ │ │ ├── data.json │ │ │ └── index.js │ │ ├── settings/ │ │ │ ├── data.json │ │ │ └── index.js │ │ └── tags/ │ │ ├── data.json │ │ └── index.js │ ├── clean-dev-storage.sh │ ├── conf/ │ │ └── proxy-api.conf │ ├── docker-compose.staging.yml │ ├── docker-compose.yml │ ├── drone/ │ │ ├── build_images.sh │ │ ├── files_changed.sh │ │ ├── get_go_dependencies.sh │ │ ├── get_py_dependencies.sh │ │ ├── test_front.sh │ │ ├── test_go.sh │ │ └── test_py.sh │ ├── extract/ │ │ └── email_graph.py │ ├── fixtures/ │ │ ├── mbox/ │ │ │ ├── dev@caliopen.local/ │ │ │ │ ├── 1460463561.5266_1.argentina:2, │ │ │ │ ├── 1460463561.5266_11.argentina:2, │ │ │ │ ├── 1460463561.5266_13.argentina:2, │ │ │ │ ├── 1460463561.5266_15.argentina:2, │ │ │ │ ├── 1460463561.5266_17.argentina:2, │ │ │ │ ├── 1460463561.5266_19.argentina:2, │ │ │ │ ├── 1460463561.5266_21.argentina:2, │ │ │ │ ├── 1460463561.5266_23.argentina:2, │ │ │ │ ├── 1460463561.5266_25.argentina:2, │ │ │ │ ├── 1460463561.5266_27.argentina:2, │ │ │ │ ├── 1460463561.5266_29.argentina:2, │ │ │ │ ├── 1460463561.5266_3.argentina:2, │ │ │ │ ├── 1460463561.5266_31.argentina:2, │ │ │ │ ├── 1460463561.5266_33.argentina:2, │ │ │ │ ├── 1460463561.5266_35.argentina:2, │ │ │ │ ├── 1460463561.5266_37.argentina:2, │ │ │ │ ├── 1460463561.5266_39.argentina:2, │ │ │ │ ├── 1460463561.5266_41.argentina:2, │ │ │ │ ├── 1460463561.5266_43.argentina:2, │ │ │ │ ├── 1460463561.5266_45.argentina:2, │ │ │ │ ├── 1460463561.5266_47.argentina:2, │ │ │ │ ├── 1460463561.5266_49.argentina:2, │ │ │ │ ├── 1460463561.5266_5.argentina:2, │ │ │ │ ├── 1460463561.5266_51.argentina:2, │ │ │ │ ├── 1460463561.5266_53.argentina:2, │ │ │ │ ├── 1460463561.5266_55.argentina:2, │ │ │ │ ├── 1460463561.5266_57.argentina:2, │ │ │ │ ├── 1460463561.5266_59.argentina:2, │ │ │ │ ├── 1460463561.5266_61.argentina:2, │ │ │ │ ├── 1460463561.5266_63.argentina:2, │ │ │ │ ├── 1460463561.5266_7.argentina:2, │ │ │ │ ├── 1460463561.5266_9.argentina:2, │ │ │ │ ├── 1460463562.5266_101.argentina:2, │ │ │ │ ├── 1460463562.5266_103.argentina:2, │ │ │ │ ├── 1460463562.5266_105.argentina:2, │ │ │ │ ├── 1460463562.5266_107.argentina:2, │ │ │ │ ├── 1460463562.5266_109.argentina:2, │ │ │ │ ├── 1460463562.5266_111.argentina:2, │ │ │ │ ├── 1460463562.5266_113.argentina:2, │ │ │ │ ├── 1460463562.5266_115.argentina:2, │ │ │ │ ├── 1460463562.5266_117.argentina:2, │ │ │ │ ├── 1460463562.5266_119.argentina:2, │ │ │ │ ├── 1460463562.5266_121.argentina:2, │ │ │ │ ├── 1460463562.5266_123.argentina:2, │ │ │ │ ├── 1460463562.5266_125.argentina:2, │ │ │ │ ├── 1460463562.5266_127.argentina:2, │ │ │ │ ├── 1460463562.5266_129.argentina:2, │ │ │ │ ├── 1460463562.5266_131.argentina:2, │ │ │ │ ├── 1460463562.5266_133.argentina:2, │ │ │ │ ├── 1460463562.5266_135.argentina:2, │ │ │ │ ├── 1460463562.5266_137.argentina:2, │ │ │ │ ├── 1460463562.5266_139.argentina:2, │ │ │ │ ├── 1460463562.5266_141.argentina:2, │ │ │ │ ├── 1460463562.5266_143.argentina:2, │ │ │ │ ├── 1460463562.5266_145.argentina:2, │ │ │ │ ├── 1460463562.5266_147.argentina:2, │ │ │ │ ├── 1460463562.5266_149.argentina:2, │ │ │ │ ├── 1460463562.5266_151.argentina:2, │ │ │ │ ├── 1460463562.5266_153.argentina:2, │ │ │ │ ├── 1460463562.5266_65.argentina:2, │ │ │ │ ├── 1460463562.5266_67.argentina:2, │ │ │ │ ├── 1460463562.5266_69.argentina:2, │ │ │ │ ├── 1460463562.5266_71.argentina:2, │ │ │ │ ├── 1460463562.5266_73.argentina:2, │ │ │ │ ├── 1460463562.5266_75.argentina:2, │ │ │ │ ├── 1460463562.5266_77.argentina:2, │ │ │ │ ├── 1460463562.5266_79.argentina:2, │ │ │ │ ├── 1460463562.5266_81.argentina:2, │ │ │ │ ├── 1460463562.5266_83.argentina:2, │ │ │ │ ├── 1460463562.5266_85.argentina:2, │ │ │ │ ├── 1460463562.5266_87.argentina:2, │ │ │ │ ├── 1460463562.5266_89.argentina:2, │ │ │ │ ├── 1460463562.5266_91.argentina:2, │ │ │ │ ├── 1460463562.5266_93.argentina:2, │ │ │ │ ├── 1460463562.5266_95.argentina:2, │ │ │ │ ├── 1460463562.5266_97.argentina:2, │ │ │ │ ├── 1460463562.5266_99.argentina:2, │ │ │ │ ├── 1460463563.5266_155.argentina:2, │ │ │ │ ├── 1460463563.5266_157.argentina:2, │ │ │ │ ├── 1460463563.5266_159.argentina:2, │ │ │ │ ├── 1460463563.5266_161.argentina:2, │ │ │ │ ├── 1460463563.5266_163.argentina:2, │ │ │ │ ├── 1460463563.5266_165.argentina:2, │ │ │ │ ├── 1460463563.5266_167.argentina:2, │ │ │ │ ├── 1460463563.5266_169.argentina:2, │ │ │ │ ├── 1460463563.5266_171.argentina:2, │ │ │ │ ├── 1460463563.5266_173.argentina:2, │ │ │ │ ├── 1460463563.5266_175.argentina:2, │ │ │ │ ├── 1460463563.5266_177.argentina:2, │ │ │ │ ├── 1460463563.5266_179.argentina:2, │ │ │ │ ├── 1460463563.5266_181.argentina:2, │ │ │ │ ├── 1460463563.5266_183.argentina:2, │ │ │ │ ├── 1460463563.5266_185.argentina:2, │ │ │ │ ├── 1460463563.5266_187.argentina:2, │ │ │ │ ├── 1460463563.5266_189.argentina:2, │ │ │ │ ├── 1460463563.5266_193.argentina:2, │ │ │ │ ├── 1460463563.5266_195.argentina:2, │ │ │ │ ├── 1460463563.5266_197.argentina:2, │ │ │ │ ├── 1460463563.5266_199.argentina:2, │ │ │ │ ├── 1460463563.5266_201.argentina:2, │ │ │ │ ├── 1460463563.5266_205.argentina:2, │ │ │ │ ├── 1460463563.5266_207.argentina:2, │ │ │ │ ├── 1460463563.5266_209.argentina:2, │ │ │ │ ├── 1460463563.5266_211.argentina:2, │ │ │ │ ├── 1460463563.5266_213.argentina:2, │ │ │ │ ├── 1460463563.5266_215.argentina:2, │ │ │ │ ├── 1460463563.5266_217.argentina:2, │ │ │ │ ├── 1460463563.5266_219.argentina:2, │ │ │ │ ├── 1460463563.5266_221.argentina:2, │ │ │ │ ├── 1460463563.5266_223.argentina:2, │ │ │ │ ├── 1460463563.5266_225.argentina:2, │ │ │ │ ├── 1460463563.5266_227.argentina:2, │ │ │ │ ├── 1460463563.5266_229.argentina:2, │ │ │ │ ├── 1460463563.5266_231.argentina:2, │ │ │ │ ├── 1460463563.5266_233.argentina:2, │ │ │ │ ├── 1460463563.5266_235.argentina:2, │ │ │ │ ├── 1460463563.5266_237.argentina:2, │ │ │ │ ├── 1460463563.5266_239.argentina:2, │ │ │ │ ├── 1460463563.5266_241.argentina:2, │ │ │ │ ├── 1460463563.5266_243.argentina:2, │ │ │ │ ├── 1460463564.5266_245.argentina:2, │ │ │ │ ├── 1460463564.5266_247.argentina:2, │ │ │ │ ├── 1460463564.5266_249.argentina:2, │ │ │ │ ├── 1460463564.5266_253.argentina:2, │ │ │ │ ├── 1460463564.5266_255.argentina:2, │ │ │ │ ├── 1460463564.5266_257.argentina:2, │ │ │ │ ├── 1460463564.5266_259.argentina:2, │ │ │ │ ├── 1460463564.5266_261.argentina:2, │ │ │ │ ├── 1460463564.5266_263.argentina:2, │ │ │ │ ├── 1460463564.5266_265.argentina:2, │ │ │ │ ├── 1460463564.5266_267.argentina:2, │ │ │ │ ├── 1460463564.5266_269.argentina:2, │ │ │ │ ├── 1460463564.5266_271.argentina:2, │ │ │ │ ├── 1460463564.5266_273.argentina:2, │ │ │ │ ├── 1460463564.5266_275.argentina:2, │ │ │ │ ├── 1460463564.5266_277.argentina:2, │ │ │ │ ├── 1460463564.5266_279.argentina:2, │ │ │ │ ├── 1460463564.5266_281.argentina:2, │ │ │ │ ├── 1460463564.5266_283.argentina:2, │ │ │ │ ├── 1460463564.5266_285.argentina:2, │ │ │ │ ├── 1460463564.5266_287.argentina:2, │ │ │ │ ├── 1460463564.5266_289.argentina:2, │ │ │ │ ├── 1460463564.5266_291.argentina:2, │ │ │ │ ├── 1460463564.5266_293.argentina:2, │ │ │ │ ├── 1460463564.5266_295.argentina:2, │ │ │ │ ├── 1460463564.5266_297.argentina:2, │ │ │ │ ├── 1460463564.5266_299.argentina:2, │ │ │ │ ├── 1460463564.5266_301.argentina:2, │ │ │ │ ├── 1460463564.5266_303.argentina:2, │ │ │ │ ├── 1460463564.5266_305.argentina:2, │ │ │ │ ├── 1460463564.5266_307.argentina:2, │ │ │ │ ├── 1460463564.5266_309.argentina:2, │ │ │ │ ├── 1460463564.5266_311.argentina:2, │ │ │ │ ├── 1460463564.5266_313.argentina:2, │ │ │ │ ├── 1460463564.5266_315.argentina:2, │ │ │ │ ├── 1460463564.5266_317.argentina:2, │ │ │ │ ├── 1460463564.5266_319.argentina:2, │ │ │ │ ├── 1460463564.5266_321.argentina:2, │ │ │ │ ├── 1460463564.5266_323.argentina:2, │ │ │ │ ├── 1460463564.5266_325.argentina:2, │ │ │ │ ├── 1460463564.5266_327.argentina:2, │ │ │ │ ├── 1460463564.5266_329.argentina:2, │ │ │ │ ├── 1460463564.5266_331.argentina:2, │ │ │ │ ├── 1460463564.5266_333.argentina:2, │ │ │ │ ├── 1460463564.5266_335.argentina:2, │ │ │ │ ├── 1460463564.5266_337.argentina:2, │ │ │ │ ├── 1460463564.5266_339.argentina:2, │ │ │ │ ├── 1460463564.5266_341.argentina:2, │ │ │ │ ├── 1460463564.5266_343.argentina:2, │ │ │ │ ├── 1460463565.5266_345.argentina:2, │ │ │ │ ├── 1460463565.5266_347.argentina:2, │ │ │ │ ├── 1460463565.5266_349.argentina:2, │ │ │ │ ├── 1460463565.5266_351.argentina:2, │ │ │ │ ├── 1460463565.5266_353.argentina:2, │ │ │ │ ├── 1460463565.5266_355.argentina:2, │ │ │ │ ├── 1460463565.5266_357.argentina:2, │ │ │ │ ├── 1460463565.5266_359.argentina:2, │ │ │ │ ├── 1460463565.5266_361.argentina:2, │ │ │ │ ├── 1460463565.5266_363.argentina:2, │ │ │ │ ├── 1460463565.5266_365.argentina:2, │ │ │ │ ├── 1460463565.5266_367.argentina:2, │ │ │ │ ├── 1460463565.5266_369.argentina:2, │ │ │ │ ├── 1460463565.5266_371.argentina:2, │ │ │ │ ├── 1460463565.5266_373.argentina:2, │ │ │ │ ├── 1460463565.5266_375.argentina:2, │ │ │ │ ├── 1460463565.5266_377.argentina:2, │ │ │ │ ├── 1460463565.5266_379.argentina:2, │ │ │ │ ├── 1460463565.5266_381.argentina:2, │ │ │ │ ├── 1460463565.5266_383.argentina:2, │ │ │ │ ├── 1460463565.5266_385.argentina:2, │ │ │ │ ├── 1460463565.5266_387.argentina:2, │ │ │ │ ├── 1460463565.5266_389.argentina:2, │ │ │ │ ├── 1460463565.5266_391.argentina:2, │ │ │ │ ├── 1460463565.5266_393.argentina:2, │ │ │ │ ├── 1460463565.5266_395.argentina:2, │ │ │ │ ├── 1460463565.5266_397.argentina:2, │ │ │ │ ├── 1460463565.5266_399.argentina:2, │ │ │ │ ├── 1460463565.5266_401.argentina:2, │ │ │ │ ├── 1460463565.5266_403.argentina:2, │ │ │ │ ├── 1460463565.5266_405.argentina:2, │ │ │ │ ├── 1460463565.5266_407.argentina:2, │ │ │ │ ├── 1460463565.5266_409.argentina:2, │ │ │ │ ├── 1460463565.5266_411.argentina:2, │ │ │ │ ├── 1460463565.5266_413.argentina:2, │ │ │ │ ├── 1460463565.5266_415.argentina:2, │ │ │ │ ├── 1460463565.5266_417.argentina:2, │ │ │ │ ├── 1460463565.5266_419.argentina:2, │ │ │ │ ├── 1460463565.5266_421.argentina:2, │ │ │ │ ├── 1460463565.5266_423.argentina:2, │ │ │ │ ├── 1460463565.5266_425.argentina:2, │ │ │ │ ├── 1460463565.5266_427.argentina:2, │ │ │ │ ├── 1460463565.5266_429.argentina:2, │ │ │ │ ├── 1460463565.5266_431.argentina:2, │ │ │ │ ├── 1460463565.5266_433.argentina:2, │ │ │ │ ├── 1460463565.5266_435.argentina:2, │ │ │ │ ├── 1460463566.5266_437.argentina:2, │ │ │ │ ├── 1460463566.5266_439.argentina:2, │ │ │ │ ├── 1460463566.5266_441.argentina:2, │ │ │ │ ├── 1460463566.5266_443.argentina:2, │ │ │ │ ├── 1460463566.5266_445.argentina:2, │ │ │ │ ├── 1460463566.5266_447.argentina:2, │ │ │ │ ├── 1460463566.5266_449.argentina:2, │ │ │ │ ├── 1460463566.5266_451.argentina:2, │ │ │ │ ├── 1460463566.5266_453.argentina:2, │ │ │ │ ├── 1460463566.5266_455.argentina:2, │ │ │ │ ├── 1460463566.5266_457.argentina:2, │ │ │ │ ├── 1460463566.5266_459.argentina:2, │ │ │ │ ├── 1460463566.5266_461.argentina:2, │ │ │ │ ├── 1460463566.5266_463.argentina:2, │ │ │ │ ├── 1460463566.5266_465.argentina:2, │ │ │ │ ├── 1460463566.5266_467.argentina:2, │ │ │ │ ├── 1460463566.5266_469.argentina:2, │ │ │ │ ├── 1460463566.5266_471.argentina:2, │ │ │ │ ├── 1460463566.5266_473.argentina:2, │ │ │ │ ├── 1460463566.5266_475.argentina:2, │ │ │ │ ├── 1460463567.5266_477.argentina:2, │ │ │ │ ├── 1460463567.5266_483.argentina:2, │ │ │ │ ├── 1460463567.5266_485.argentina:2, │ │ │ │ ├── 1460463567.5266_487.argentina:2, │ │ │ │ ├── 1460463567.5266_489.argentina:2, │ │ │ │ ├── 1460463567.5266_491.argentina:2, │ │ │ │ ├── 1460463567.5266_493.argentina:2, │ │ │ │ ├── 1460463567.5266_495.argentina:2, │ │ │ │ ├── 1460463567.5266_497.argentina:2, │ │ │ │ ├── 1460463567.5266_499.argentina:2, │ │ │ │ ├── 1460463567.5266_503.argentina:2, │ │ │ │ ├── 1460463567.5266_505.argentina:2, │ │ │ │ ├── 1460463567.5266_507.argentina:2, │ │ │ │ ├── 1460463567.5266_509.argentina:2, │ │ │ │ ├── 1460463567.5266_511.argentina:2, │ │ │ │ ├── 1460463567.5266_513.argentina:2, │ │ │ │ ├── 1460463567.5266_515.argentina:2, │ │ │ │ ├── 1460463567.5266_517.argentina:2, │ │ │ │ ├── 1460463567.5266_519.argentina:2, │ │ │ │ ├── 1460463567.5266_521.argentina:2, │ │ │ │ ├── 1460463567.5266_525.argentina:2, │ │ │ │ ├── 1460463567.5266_527.argentina:2, │ │ │ │ ├── 1460463567.5266_529.argentina:2, │ │ │ │ ├── 1460463567.5266_531.argentina:2, │ │ │ │ ├── 1460463568.5266_533.argentina:2, │ │ │ │ ├── 1460463568.5266_535.argentina:2, │ │ │ │ ├── 1460463568.5266_537.argentina:2, │ │ │ │ ├── 1460463568.5266_539.argentina:2, │ │ │ │ ├── 1460463568.5266_541.argentina:2, │ │ │ │ ├── 1460463568.5266_543.argentina:2, │ │ │ │ ├── 1460463568.5266_545.argentina:2, │ │ │ │ ├── 1460463568.5266_547.argentina:2, │ │ │ │ ├── 1460463568.5266_549.argentina:2, │ │ │ │ ├── 1460463568.5266_551.argentina:2, │ │ │ │ ├── 1460463568.5266_553.argentina:2, │ │ │ │ ├── 1460463568.5266_555.argentina:2, │ │ │ │ ├── 1460463568.5266_557.argentina:2, │ │ │ │ ├── 1460463568.5266_559.argentina:2, │ │ │ │ ├── 1460463568.5266_561.argentina:2, │ │ │ │ ├── 1460463568.5266_563.argentina:2, │ │ │ │ ├── 1460463568.5266_565.argentina:2, │ │ │ │ ├── 1460463568.5266_567.argentina:2, │ │ │ │ ├── 1460463568.5266_569.argentina:2, │ │ │ │ ├── 1460463568.5266_571.argentina:2, │ │ │ │ ├── 1460463568.5266_573.argentina:2, │ │ │ │ ├── 1460463568.5266_575.argentina:2, │ │ │ │ ├── 1460463568.5266_577.argentina:2, │ │ │ │ ├── 1460463568.5266_579.argentina:2, │ │ │ │ ├── 1460463568.5266_581.argentina:2, │ │ │ │ ├── 1460463568.5266_583.argentina:2, │ │ │ │ ├── 1460463568.5266_585.argentina:2, │ │ │ │ ├── 1460463568.5266_587.argentina:2, │ │ │ │ ├── 1460463568.5266_589.argentina:2, │ │ │ │ ├── 1460463568.5266_591.argentina:2, │ │ │ │ ├── 1460463568.5266_593.argentina:2, │ │ │ │ ├── 1460463568.5266_595.argentina:2, │ │ │ │ ├── 1460463568.5266_597.argentina:2, │ │ │ │ ├── 1460463568.5266_599.argentina:2, │ │ │ │ ├── 1460463568.5266_601.argentina:2, │ │ │ │ ├── 1460463568.5266_603.argentina:2, │ │ │ │ ├── 1460463568.5266_605.argentina:2, │ │ │ │ ├── 1460463568.5266_607.argentina:2, │ │ │ │ ├── 1460463568.5266_609.argentina:2, │ │ │ │ ├── 1460463568.5266_611.argentina:2, │ │ │ │ ├── 1460463568.5266_613.argentina:2, │ │ │ │ ├── 1460463568.5266_615.argentina:2, │ │ │ │ ├── 1460463568.5266_617.argentina:2, │ │ │ │ ├── 1460463569.5266_619.argentina:2, │ │ │ │ ├── 1460463569.5266_621.argentina:2, │ │ │ │ ├── 1460463569.5266_623.argentina:2, │ │ │ │ ├── 1460463569.5266_625.argentina:2, │ │ │ │ ├── 1460463569.5266_627.argentina:2, │ │ │ │ ├── 1460463569.5266_629.argentina:2, │ │ │ │ ├── 1460463569.5266_631.argentina:2, │ │ │ │ ├── 1460463569.5266_633.argentina:2, │ │ │ │ ├── 1460463569.5266_635.argentina:2, │ │ │ │ ├── 1460463569.5266_637.argentina:2, │ │ │ │ ├── 1460463569.5266_639.argentina:2, │ │ │ │ ├── 1460463569.5266_641.argentina:2, │ │ │ │ ├── 1460463569.5266_643.argentina:2, │ │ │ │ ├── 1460463569.5266_645.argentina:2, │ │ │ │ ├── 1460463569.5266_647.argentina:2, │ │ │ │ ├── 1460463569.5266_649.argentina:2, │ │ │ │ ├── 1460463569.5266_651.argentina:2, │ │ │ │ ├── 1460463569.5266_653.argentina:2, │ │ │ │ ├── 1460463569.5266_655.argentina:2, │ │ │ │ ├── 1460463569.5266_657.argentina:2, │ │ │ │ ├── 1460463569.5266_659.argentina:2, │ │ │ │ ├── 1460463569.5266_661.argentina:2, │ │ │ │ ├── 1460463569.5266_663.argentina:2, │ │ │ │ ├── 1460463569.5266_665.argentina:2, │ │ │ │ ├── 1460463569.5266_667.argentina:2, │ │ │ │ ├── 1460463569.5266_669.argentina:2, │ │ │ │ ├── 1460463569.5266_671.argentina:2, │ │ │ │ ├── 1460463569.5266_673.argentina:2, │ │ │ │ ├── 1460463569.5266_675.argentina:2, │ │ │ │ ├── 1460463569.5266_677.argentina:2, │ │ │ │ ├── 1460463569.5266_679.argentina:2, │ │ │ │ ├── 1460463569.5266_681.argentina:2, │ │ │ │ ├── 1460463569.5266_683.argentina:2, │ │ │ │ ├── 1460463569.5266_685.argentina:2, │ │ │ │ ├── 1460463569.5266_687.argentina:2, │ │ │ │ ├── 1460463569.5266_689.argentina:2, │ │ │ │ ├── 1460463569.5266_691.argentina:2, │ │ │ │ ├── 1460463569.5266_693.argentina:2, │ │ │ │ ├── 1460463569.5266_695.argentina:2, │ │ │ │ ├── 1460463569.5266_697.argentina:2, │ │ │ │ ├── 1460463569.5266_699.argentina:2, │ │ │ │ ├── 1460463569.5266_701.argentina:2, │ │ │ │ ├── 1460463569.5266_703.argentina:2, │ │ │ │ ├── 1460463569.5266_705.argentina:2, │ │ │ │ ├── 1460463570.5266_707.argentina:2, │ │ │ │ ├── 1460463570.5266_709.argentina:2, │ │ │ │ ├── 1460463570.5266_711.argentina:2, │ │ │ │ ├── 1460463570.5266_713.argentina:2, │ │ │ │ ├── 1460463570.5266_715.argentina:2, │ │ │ │ ├── 1460463570.5266_717.argentina:2, │ │ │ │ ├── 1460463570.5266_719.argentina:2, │ │ │ │ ├── 1460463570.5266_721.argentina:2, │ │ │ │ ├── 1460463570.5266_723.argentina:2, │ │ │ │ ├── 1460463570.5266_725.argentina:2, │ │ │ │ ├── 1460463570.5266_727.argentina:2, │ │ │ │ ├── 1460463570.5266_729.argentina:2, │ │ │ │ ├── 1460463570.5266_731.argentina:2, │ │ │ │ ├── 1460463570.5266_733.argentina:2, │ │ │ │ ├── 1460463570.5266_735.argentina:2, │ │ │ │ ├── 1460463570.5266_737.argentina:2, │ │ │ │ ├── 1460463570.5266_739.argentina:2, │ │ │ │ ├── 1460463570.5266_741.argentina:2, │ │ │ │ ├── 1460463570.5266_743.argentina:2, │ │ │ │ ├── 1460463570.5266_745.argentina:2, │ │ │ │ ├── 1460463570.5266_747.argentina:2, │ │ │ │ ├── 1460463570.5266_749.argentina:2, │ │ │ │ ├── 1460463570.5266_751.argentina:2, │ │ │ │ ├── 1460463570.5266_753.argentina:2, │ │ │ │ ├── 1460463570.5266_755.argentina:2, │ │ │ │ ├── 1460463570.5266_757.argentina:2, │ │ │ │ ├── 1460463570.5266_759.argentina:2, │ │ │ │ ├── 1460463570.5266_761.argentina:2, │ │ │ │ ├── 1460463570.5266_763.argentina:2, │ │ │ │ ├── 1460463570.5266_765.argentina:2, │ │ │ │ ├── 1460463570.5266_767.argentina:2, │ │ │ │ ├── 1460463570.5266_769.argentina:2, │ │ │ │ ├── 1460463570.5266_771.argentina:2, │ │ │ │ ├── 1460463570.5266_773.argentina:2, │ │ │ │ ├── 1460463570.5266_775.argentina:2, │ │ │ │ ├── 1460463570.5266_777.argentina:2, │ │ │ │ ├── 1460463570.5266_779.argentina:2, │ │ │ │ ├── 1460463570.5266_781.argentina:2, │ │ │ │ ├── 1460463571.5266_783.argentina:2, │ │ │ │ ├── 1460463571.5266_785.argentina:2, │ │ │ │ ├── 1460463571.5266_787.argentina:2, │ │ │ │ ├── 1460463571.5266_789.argentina:2, │ │ │ │ ├── 1460463571.5266_791.argentina:2, │ │ │ │ ├── 1460463571.5266_793.argentina:2, │ │ │ │ ├── 1460463571.5266_795.argentina:2, │ │ │ │ ├── 1460463571.5266_797.argentina:2, │ │ │ │ ├── 1460463571.5266_799.argentina:2, │ │ │ │ ├── 1460463571.5266_801.argentina:2, │ │ │ │ ├── Re: [Caliopdev] CR réunion du 12_12_2014 - David Epely - 2014-12-13 1352.eml │ │ │ │ ├── Re: [Caliopdev] Compte rendu reunion 21 Novembre - Laurent Chemla - 2014-11-22 1046.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - Aymeric Barantal - 2014-11-26 1605.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - Laurent Chemla - 2014-11-26 2346.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - julien muetton - 2014-11-28 1701.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - Alban Crommer - 2014-11-27 1007.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - Alban Crommer - 2014-11-28 1625.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - Alexis - 2014-11-28 1518.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - Alexis - 2014-11-28 1748.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - Aymeric Barantal - 2014-11-28 1112.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - Aymeric Barantal - 2014-11-28 1724.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - Kajan Sivaramalingam - 2014-11-27 0132.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - Vincent MINEAUD - 2014-11-28 1600.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - joseph@cozycloud.cc - 2014-11-27 1120.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - joseph@cozycloud.cc - 2014-11-27 1228.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - julien muetton - 2014-11-27 1211.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - julien muetton - 2014-11-27 2140.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - julien muetton - 2014-11-28 1222.eml │ │ │ │ ├── Re: [Caliopdev] Fwd: Re: matrice de choix framework JS - julien muetton - 2014-11-28 1609.eml │ │ │ │ ├── Re: [Caliopdev] Nouveau screencast (brut) - Laurent Chemla - 2014-11-30 1724.eml │ │ │ │ ├── Re: [Caliopdev] Nouveau screencast (brut) - Laurent Chemla - 2014-12-01 1214.eml │ │ │ │ ├── Re: [Caliopdev] Nouveau screencast (brut) - Thomas Laurent - 2014-12-01 1149.eml │ │ │ │ ├── Re: [Caliopdev] Prochaine réunion - Aymeric Barantal - 2014-12-02 1510.eml │ │ │ │ ├── Re: [Caliopdev] Prochaine réunion - David Epely - 2014-12-02 1056.eml │ │ │ │ ├── Re: [Caliopdev] Prochaine réunion - Kajan Sivaramalingam - 2014-12-01 2345.eml │ │ │ │ ├── Re: [Caliopdev] Prochaine réunion - Kajan Sivaramalingam - 2014-12-02 2319.eml │ │ │ │ ├── Re: [Caliopdev] Prochaine réunion - Kajan Sivaramalingam - 2014-12-02 2322.eml │ │ │ │ ├── Re: [Caliopdev] Prochaine réunion - Laurent Chemla - 2014-12-02 0610.eml │ │ │ │ ├── Re: [Caliopdev] Prochaine réunion - Laurent Chemla - 2014-12-02 1158.eml │ │ │ │ ├── Re: [Caliopdev] Prochaine réunion - julien muetton - 2014-12-02 1031.eml │ │ │ │ ├── Re: [Caliopdev] Prochaines réunions. - Alban Crommer - 2014-12-29 1917.eml │ │ │ │ ├── Re: [Caliopdev] Prochaines étapes ? - Laurent Chemla - 2014-12-06 2135.eml │ │ │ │ ├── Re: [Caliopdev] [svetlana.meyer@ensc.fr: Bravo!] - Alexis - 2014-11-26 1446.eml │ │ │ │ ├── Re: [Caliopdev] framework js - David Epely - 2014-11-21 0920.eml │ │ │ │ ├── Re: [Caliopdev] framework js - Kajan Sivaramalingam - 2014-11-21 0059.eml │ │ │ │ ├── [Caliopdev] CR réunion du 12_12_2014 - Thomas Laurent - 2014-12-12 1701.eml │ │ │ │ ├── [Caliopdev] Compte rendu reunion 21 Novembre - Aymeric Barantal - 2014-11-21 1221.eml │ │ │ │ ├── [Caliopdev] Fwd: Re: matrice de choix framework JS - Aymeric Barantal - 2014-11-25 1119.eml │ │ │ │ ├── [Caliopdev] Nouveau screencast (brut) - Aymeric Barantal - 2014-11-28 1718.eml │ │ │ │ ├── [Caliopdev] Prochaine réunion - Laurent Chemla - 2014-12-01 2142.eml │ │ │ │ ├── [Caliopdev] Prochaines réunions. - Laurent Chemla - 2014-12-29 1807.eml │ │ │ │ ├── [Caliopdev] Prochaines étapes ? - Kajan Sivaramalingam - 2014-12-06 2108.eml │ │ │ │ ├── [Caliopdev] [Coin-coin@canapin.com: Petites fautes dans la FAQ] - Laurent Chemla - 2014-11-24 1840.eml │ │ │ │ ├── [Caliopdev] [benjamin@sonntag.fr: Fwd: ***UNCHECKED*** Caliopen FAQ] - Laurent Chemla - 2014-12-15 1825.eml │ │ │ │ ├── [Caliopdev] [contact+caliopen@etiennesamson.com: Intégration] - Laurent Chemla - 2014-11-24 1840.eml │ │ │ │ ├── [Caliopdev] [groupeiw@gmail.com: Contacter l'équipe CaliOpen] - Laurent Chemla - 2014-11-25 1810.eml │ │ │ │ ├── [Caliopdev] [svetlana.meyer@ensc.fr: Bravo!] - Laurent Chemla - 2014-11-26 1244.eml │ │ │ │ ├── [Caliopdev] [svetlana.meyer@ensc.fr: Re: Bravo!] - Laurent Chemla - 2014-11-26 1246.eml │ │ │ │ └── [Caliopdev] matrice de choix framework JS - Aymeric Barantal - 2014-11-24 1146.eml │ │ │ └── test@caliopen.local/ │ │ │ ├── test_1.eml │ │ │ ├── test_2.eml │ │ │ └── test_3.eml │ │ ├── messages_body/ │ │ │ ├── flat-html-body │ │ │ ├── html-mailinglist │ │ │ ├── html-with-inline-styles │ │ │ └── spam-htmj-and-styles │ │ ├── raw_emails/ │ │ │ ├── email-with-iso-8859-1-encoding │ │ │ ├── html-in-plain-body │ │ │ ├── html-with-inlined-image │ │ │ ├── invalid-utf-8 │ │ │ ├── multipart-html │ │ │ ├── multipart-signed │ │ │ ├── multipart-with-attachment │ │ │ ├── sample-mail.eml.txt │ │ │ ├── signed-and-encrypted │ │ │ ├── simple │ │ │ ├── spam-multipart │ │ │ ├── with-docx-attachment │ │ │ └── with-json-attachment │ │ ├── raw_inbound_msg.cql │ │ ├── twitter/ │ │ │ ├── dm-multimedia.json │ │ │ └── dm.json │ │ └── vcard/ │ │ ├── multi.vcf │ │ ├── rfc2425-1.vcard │ │ ├── rfc2425-2.vcard │ │ ├── rfc2425-3.vcard │ │ ├── rfc2426-1.vcard │ │ ├── rfc2426-2.vcard │ │ ├── rfc2426-3.vcard │ │ ├── rfc2426-4.vcard │ │ └── rfc2426-5.vcard │ ├── gen-swagger-spec.sh │ ├── init.sh │ ├── kubernetes/ │ │ ├── README.md │ │ ├── clean-minikube.sh │ │ ├── configs/ │ │ │ └── dns-config.yaml │ │ ├── deploy-minikube.sh │ │ ├── deployments/ │ │ │ ├── apiv1-deployment.yaml │ │ │ ├── apiv2-deployment.yaml │ │ │ ├── cassandra-deployment.yaml │ │ │ ├── elasticsearch-deployment.yaml │ │ │ ├── frontend-deployment.yaml │ │ │ ├── identity-poller-deployment.yaml │ │ │ ├── imap-worker-deployment.yaml │ │ │ ├── lmtpd-deployment.yaml │ │ │ ├── message-handler-deployment.yaml │ │ │ ├── minio-deployment.yaml │ │ │ ├── nats-deployment.yaml │ │ │ ├── redis-deployment.yaml │ │ │ └── smtp-deployment.yaml │ │ ├── jobs/ │ │ │ ├── cli-admin-creation.yaml │ │ │ ├── cli-dev-creation.yaml │ │ │ ├── cli-mail-import.yaml │ │ │ └── cli-setup.yaml │ │ ├── services/ │ │ │ ├── apiv1-service.yaml │ │ │ ├── apiv2-service.yaml │ │ │ ├── cassandra-service.yaml │ │ │ ├── elasticsearch-service.yaml │ │ │ ├── external-go-service.yaml │ │ │ ├── external-python-service.yaml │ │ │ ├── frontend-service.yaml │ │ │ ├── lmtpd-service.yaml │ │ │ ├── minio-service.yaml │ │ │ ├── nats-service.yaml │ │ │ ├── redis-service.yaml │ │ │ └── smtp-service.yaml │ │ └── volumeclaims/ │ │ ├── db-persistentvolumeclaim.yaml │ │ ├── index-persistentvolumeclaim.yaml │ │ └── store-persistentvolumeclaim.yaml │ ├── make_release │ ├── makefile │ ├── manage_package │ ├── manage_package-requirements.txt │ ├── migrations/ │ │ ├── ES_mappings_v2.json │ │ ├── add_delivered_column_to_raw-message_table.cql │ │ ├── add_temp_id_to_attachment_type.cql │ │ ├── fill_message_is_received.py │ │ ├── index_migration_v0_to_v2.py │ │ ├── index_migration_v2_to_v3.py │ │ ├── index_migration_v3_to_v4.py │ │ ├── index_migration_v4_to_v5.py │ │ ├── index_migration_v5_to_v6.py │ │ ├── set_date_sort_in_db.py │ │ └── update_ecdsa_key.py │ ├── packages.yaml │ ├── publish-images.sh │ ├── registry.conf.template │ ├── run-tests.sh │ ├── setup-virtualenv.sh │ ├── start.sh │ ├── stop.sh │ └── storybook/ │ ├── .storybook/ │ │ ├── addons.js │ │ ├── config.js │ │ └── webpack.config.js │ ├── README.md │ ├── package.json │ └── stories/ │ ├── CHANGELOG.md │ ├── Changelog.jsx │ ├── Guideline/ │ │ ├── index.jsx │ │ └── style.scss │ ├── Welcome.jsx │ ├── components/ │ │ ├── BlockList.jsx │ │ ├── CollectionFieldGroup.jsx │ │ ├── ContactDetails.jsx │ │ ├── FormGrid.jsx │ │ ├── Link.jsx │ │ ├── MessageList.jsx │ │ ├── PasswordStrength.jsx │ │ ├── PiBar.jsx │ │ ├── Reply.jsx │ │ ├── Subtitle.jsx │ │ └── TagsForm.jsx │ ├── index.jsx │ ├── layouts/ │ │ ├── ContactBook.jsx │ │ ├── Devices.jsx │ │ ├── SigninPage.jsx │ │ └── SignupPage.jsx │ └── presenters/ │ ├── Code.jsx │ ├── ComponentWrapper.jsx │ └── index.js ├── doc/ │ ├── architecture/ │ │ ├── Repository_structure.md │ │ └── assets/ │ │ └── stack_frontend_dev_2017-03-09.puml │ ├── devops/ │ │ ├── continous-integration.md │ │ ├── docker-&-registry.md │ │ └── kube_survival_guide.md │ ├── index.md │ ├── install/ │ │ ├── frontend-development.md │ │ ├── minikube-local-development.md │ │ └── native-installation.md │ ├── install.md │ ├── monitoring-debug/ │ │ └── analyse-frontend-bundle.md │ ├── release/ │ │ ├── Communication.md │ │ ├── Guidelines.md │ │ ├── README.md │ │ └── deploy_kubernetes.md │ ├── specifications/ │ │ ├── attachments/ │ │ │ ├── assets/ │ │ │ │ └── management.puml │ │ │ └── index.md │ │ ├── client/ │ │ │ ├── architecture.md │ │ │ ├── frontend-data-flow.md │ │ │ └── routing.md │ │ ├── contact/ │ │ │ ├── assets/ │ │ │ │ └── discover_contact_from_email.uml │ │ │ ├── index.md │ │ │ └── vcard_doc.md │ │ ├── discussions/ │ │ │ ├── assets/ │ │ │ │ └── delete_discussion.uml │ │ │ └── delete_discussion.md │ │ ├── email-protocol/ │ │ │ ├── assets/ │ │ │ │ ├── email-delivery-inbound-rpc.puml │ │ │ │ ├── email-delivery-inbound.puml │ │ │ │ ├── email-delivery-outbound.puml │ │ │ │ ├── email-inbound-delivery-poc.puml │ │ │ │ ├── imap-outbound-process.puml │ │ │ │ └── smtp-delivery-inbound-closeup.puml │ │ │ ├── imap-binaries.md │ │ │ └── index.md │ │ ├── identities/ │ │ │ ├── Remote-identities-creation-diagram.puml │ │ │ ├── index.md │ │ │ └── remote-identities.md │ │ ├── message/ │ │ │ ├── assets/ │ │ │ │ ├── message-create-save-send.puml │ │ │ │ ├── message-create-save-send_nats_async.puml │ │ │ │ └── message_discussion_association.puml │ │ │ └── index.md │ │ ├── messages-drafts-discussions-routes/ │ │ │ └── index-fr.md │ │ ├── messaging-system/ │ │ │ └── index.md │ │ ├── notification/ │ │ │ ├── API-fr.md │ │ │ ├── assets/ │ │ │ │ └── client-message-sequence.puml │ │ │ └── frontend.md │ │ ├── patch/ │ │ │ └── index.md │ │ ├── protocol-implementation.md │ │ ├── pwa/ │ │ │ ├── assets/ │ │ │ │ └── sequence.puml │ │ │ └── pwa.md │ │ ├── remote-identities/ │ │ │ ├── assets/ │ │ │ │ └── twitter-oauth.puml │ │ │ ├── gmail.md │ │ │ ├── oauth.md │ │ │ └── twitter.md │ │ ├── rest-api/ │ │ │ └── index.md │ │ ├── scss-reference.md │ │ ├── search/ │ │ │ └── index.md │ │ ├── user-and-device-identification/ │ │ │ ├── assets/ │ │ │ │ ├── authenticate_new_device.uml │ │ │ │ ├── create_user_and_device.uml │ │ │ │ ├── user_and_device_authentication.uml │ │ │ │ └── user_update_credential.uml │ │ │ ├── index.md │ │ │ └── revoke-devices.md │ │ └── username/ │ │ └── index.md │ ├── tools.md │ └── welcome.md └── src/ ├── backend/ │ ├── Dockerfile.caliopen-go │ ├── Dockerfile.caliopen-python │ ├── Dockerfile.cli │ ├── Dockerfile.cli-ml │ ├── Dockerfile.go-api │ ├── Dockerfile.go-lmtp │ ├── Dockerfile.identity-poller │ ├── Dockerfile.imap-worker │ ├── Dockerfile.mastodon-worker │ ├── Dockerfile.mq-worker │ ├── Dockerfile.py-api │ ├── Dockerfile.twitter-worker │ ├── brokers/ │ │ ├── go.emails/ │ │ │ ├── README.md │ │ │ ├── broker.go │ │ │ ├── email.go │ │ │ ├── encrypt.go │ │ │ ├── inbound.go │ │ │ └── outbound.go │ │ ├── go.mastodon/ │ │ │ ├── broker.go │ │ │ ├── direct_messages.go │ │ │ ├── doc.go │ │ │ ├── inbound.go │ │ │ └── outbound.go │ │ └── go.twitter/ │ │ ├── broker.go │ │ ├── direct_messages.go │ │ ├── doc.go │ │ ├── inbound.go │ │ └── outbound.go │ ├── components/ │ │ ├── py.data/ │ │ │ ├── CHANGES.rst │ │ │ ├── README.rst │ │ │ ├── caliopen_data/ │ │ │ │ ├── __init__.py │ │ │ │ ├── interface.py │ │ │ │ ├── provider.py │ │ │ │ └── store.py │ │ │ ├── requirements.deps │ │ │ ├── setup.cfg │ │ │ └── setup.py │ │ ├── py.pgp/ │ │ │ ├── CHANGES.rst │ │ │ ├── MANIFEST.in │ │ │ ├── README.rst │ │ │ ├── caliopen_pgp/ │ │ │ │ ├── __init__.py │ │ │ │ └── keys/ │ │ │ │ ├── __init__.py │ │ │ │ ├── base.py │ │ │ │ ├── contact.py │ │ │ │ ├── discoverer.py │ │ │ │ ├── hkp.py │ │ │ │ ├── keybase.py │ │ │ │ └── rfc7929.py │ │ │ ├── setup.cfg │ │ │ └── setup.py │ │ ├── py.pi/ │ │ │ ├── CHANGES.rst │ │ │ ├── README.rst │ │ │ ├── caliopen_pi/ │ │ │ │ ├── __init__.py │ │ │ │ ├── features/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── contact.py │ │ │ │ │ ├── device.py │ │ │ │ │ ├── helpers/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── histogram.py │ │ │ │ │ │ ├── importance_level.py │ │ │ │ │ │ ├── ingress_path.py │ │ │ │ │ │ └── spam.py │ │ │ │ │ ├── mail.py │ │ │ │ │ └── types.py │ │ │ │ ├── qualifiers/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── base.py │ │ │ │ │ ├── contact.py │ │ │ │ │ ├── device.py │ │ │ │ │ ├── mail.py │ │ │ │ │ ├── mastodon.py │ │ │ │ │ └── twitter.py │ │ │ │ └── tests/ │ │ │ │ ├── fixtures/ │ │ │ │ │ ├── dkim1.eml │ │ │ │ │ ├── pgp_crypted_1.eml │ │ │ │ │ ├── pgpsigned1.elm │ │ │ │ │ ├── spam1.eml │ │ │ │ │ ├── spam2.eml │ │ │ │ │ └── spam3.eml │ │ │ │ ├── test_features.py │ │ │ │ ├── test_importance_level.py │ │ │ │ └── test_spam.py │ │ │ ├── requirements.deps │ │ │ ├── setup.cfg │ │ │ └── setup.py │ │ └── py.tag/ │ │ ├── CHANGES.rst │ │ ├── README.rst │ │ ├── caliopen_tag/ │ │ │ ├── __init__.py │ │ │ ├── models_manager/ │ │ │ │ ├── __init__.py │ │ │ │ ├── data_manager.py │ │ │ │ └── manager.py │ │ │ ├── taggers/ │ │ │ │ ├── __init__.py │ │ │ │ └── tagger.py │ │ │ ├── tests/ │ │ │ │ ├── test_model_manager.py │ │ │ │ └── test_tag.py │ │ │ └── utils.py │ │ ├── requirements.deps │ │ ├── setup.cfg │ │ └── setup.py │ ├── configs/ │ │ ├── apiv1-nginx.conf │ │ ├── apiv1-supervisord.conf │ │ ├── apiv1.ini │ │ ├── apiv2.yaml │ │ ├── caliopen-api.development.ini │ │ ├── caliopen.yaml │ │ ├── caliopen.yaml.template │ │ ├── idpoller.yaml │ │ ├── imapworker.yaml │ │ ├── lmtp.yaml │ │ ├── mastodonworker.yaml │ │ ├── swagger.json │ │ └── twitterworker.yaml │ ├── defs/ │ │ ├── go-objects/ │ │ │ ├── actions_payload.go │ │ │ ├── attachment.go │ │ │ ├── cache.go │ │ │ ├── common.go │ │ │ ├── configs.go │ │ │ ├── constants.go │ │ │ ├── contact.go │ │ │ ├── credentials.go │ │ │ ├── crypto_keys.go │ │ │ ├── device.go │ │ │ ├── discussion.go │ │ │ ├── email.go │ │ │ ├── errors.go │ │ │ ├── external_references.go │ │ │ ├── im.go │ │ │ ├── location.go │ │ │ ├── message.go │ │ │ ├── nats.go │ │ │ ├── notification.go │ │ │ ├── notification_test.go │ │ │ ├── organization.go │ │ │ ├── participant.go │ │ │ ├── participants_test.go │ │ │ ├── phone.go │ │ │ ├── postal_address.go │ │ │ ├── privacy_features.go │ │ │ ├── privacy_index.go │ │ │ ├── provider.go │ │ │ ├── raw_message.go │ │ │ ├── search.go │ │ │ ├── settings.go │ │ │ ├── social_identity.go │ │ │ ├── tag.go │ │ │ ├── tls.go │ │ │ ├── user.go │ │ │ ├── user_identity.go │ │ │ ├── username.go │ │ │ └── uuid.go │ │ ├── nats-messages/ │ │ │ ├── IMAPworkers_FetchOrder.yaml │ │ │ ├── SMTPqueue_ack_msg.yaml │ │ │ ├── UpdateContactPI_msg.yaml │ │ │ ├── inboundSMTP_deliver_msg.yaml │ │ │ └── outboundSMTP_deliver_msg.yaml │ │ ├── notifiers/ │ │ │ └── templates/ │ │ │ ├── email-device-validation.yaml │ │ │ ├── email-onboarding.yaml │ │ │ ├── email-reset-password-link.yaml │ │ │ └── email-welcome.yaml │ │ └── rest-api/ │ │ ├── objects/ │ │ │ ├── Actions.yaml │ │ │ ├── Attachment.yaml │ │ │ ├── Authentication.yaml │ │ │ ├── Contact.yaml │ │ │ ├── ContactIdentity.yaml │ │ │ ├── DefaultDevice.yaml │ │ │ ├── Device.yaml │ │ │ ├── DeviceLocation.yaml │ │ │ ├── Discussion.yaml │ │ │ ├── ECKey.yaml │ │ │ ├── Email.yaml │ │ │ ├── Error.yaml │ │ │ ├── ExternalReferences.yaml │ │ │ ├── IM.yaml │ │ │ ├── Identity.yaml │ │ │ ├── Message.yaml │ │ │ ├── MessageV2.yaml │ │ │ ├── NewContact.yaml │ │ │ ├── NewDevice.yaml │ │ │ ├── NewEmail.yaml │ │ │ ├── NewIM.yaml │ │ │ ├── NewMessage.yaml │ │ │ ├── NewMessageV2.yaml │ │ │ ├── NewOrganization.yaml │ │ │ ├── NewPhone.yaml │ │ │ ├── NewPostalAddress.yaml │ │ │ ├── NewPublicKey.yaml │ │ │ ├── NewSocialIdentity.yaml │ │ │ ├── NewTag.yaml │ │ │ ├── NewUser.yaml │ │ │ ├── NewUserIdentity.yaml │ │ │ ├── Notification.yaml │ │ │ ├── Organization.yaml │ │ │ ├── PI.yaml │ │ │ ├── PIMessage.yaml │ │ │ ├── Participant.yaml │ │ │ ├── ParticipantSuggest.yaml │ │ │ ├── Phone.yaml │ │ │ ├── PostalAddress.yaml │ │ │ ├── PrivacyFeatures.yaml │ │ │ ├── PublicKey.yaml │ │ │ ├── SearchResponse.yaml │ │ │ ├── Settings.yaml │ │ │ ├── ShortContact.yaml │ │ │ ├── SocialIdentity.yaml │ │ │ ├── Tag.yaml │ │ │ ├── User.yaml │ │ │ └── UserIdentity.yaml │ │ ├── paths/ │ │ │ ├── authentications.yaml │ │ │ ├── contacts.yaml │ │ │ ├── contactsV2.yaml │ │ │ ├── devices.yaml │ │ │ ├── discussions.yaml │ │ │ ├── files.yaml │ │ │ ├── hashdiscussion.yaml │ │ │ ├── identities.yaml │ │ │ ├── imports.yaml │ │ │ ├── me.yaml │ │ │ ├── messages.yaml │ │ │ ├── messagesV2.yaml │ │ │ ├── notifications.yaml │ │ │ ├── participants.yaml │ │ │ ├── passwords.yaml │ │ │ ├── providers.yaml │ │ │ ├── raws.yaml │ │ │ ├── search.yaml │ │ │ ├── settings.yaml │ │ │ ├── tags.yaml │ │ │ └── users.yaml │ │ └── swagger-root.yaml │ ├── doc/ │ │ └── api/ │ │ ├── README.md │ │ └── swagger.html │ ├── interfaces/ │ │ ├── NATS/ │ │ │ ├── go.mockednats/ │ │ │ │ └── nats.go │ │ │ └── py.client/ │ │ │ ├── CHANGES.rst │ │ │ ├── MANIFEST.in │ │ │ ├── README.rst │ │ │ ├── caliopen_nats/ │ │ │ │ ├── __init__.py │ │ │ │ ├── delivery.py │ │ │ │ ├── listener.py │ │ │ │ └── subscribers.py │ │ │ ├── requirements.deps │ │ │ ├── setup.cfg │ │ │ └── setup.py │ │ └── REST/ │ │ ├── go.server/ │ │ │ ├── README.md │ │ │ ├── api_server.go │ │ │ ├── cmd/ │ │ │ │ └── caliopen_rest/ │ │ │ │ ├── cli_cmds/ │ │ │ │ │ ├── root.go │ │ │ │ │ └── serve.go │ │ │ │ └── main.go │ │ │ ├── dump_request.go │ │ │ ├── errors.go │ │ │ ├── middlewares/ │ │ │ │ ├── authentication.go │ │ │ │ ├── config.go │ │ │ │ └── swagger.go │ │ │ ├── operations/ │ │ │ │ ├── contacts/ │ │ │ │ │ ├── Identities.go │ │ │ │ │ ├── contacts.go │ │ │ │ │ └── keys.go │ │ │ │ ├── devices/ │ │ │ │ │ └── devices.go │ │ │ │ ├── discussions/ │ │ │ │ │ └── discussions.go │ │ │ │ ├── helpers.go │ │ │ │ ├── identities/ │ │ │ │ │ └── identities.go │ │ │ │ ├── imports/ │ │ │ │ │ └── import.go │ │ │ │ ├── messages/ │ │ │ │ │ ├── actions.go │ │ │ │ │ ├── attachments.go │ │ │ │ │ └── messages.go │ │ │ │ ├── notifications/ │ │ │ │ │ └── notifications.go │ │ │ │ ├── participants/ │ │ │ │ │ ├── discussion.go │ │ │ │ │ └── suggest.go │ │ │ │ ├── providers/ │ │ │ │ │ ├── oauth-test.html │ │ │ │ │ └── providers.go │ │ │ │ ├── search.go │ │ │ │ ├── tags/ │ │ │ │ │ └── tags.go │ │ │ │ └── users/ │ │ │ │ ├── user.go │ │ │ │ └── username.go │ │ │ └── proxy.go │ │ └── py.server/ │ │ ├── CHANGES.rst │ │ ├── MANIFEST.in │ │ ├── README.rst │ │ ├── caliopen_api/ │ │ │ ├── __init__.py │ │ │ ├── base/ │ │ │ │ ├── __init__.py │ │ │ │ ├── config.py │ │ │ │ ├── context.py │ │ │ │ ├── deserializer.py │ │ │ │ ├── errors.py │ │ │ │ ├── exception.py │ │ │ │ └── renderer.py │ │ │ ├── discussion/ │ │ │ │ ├── __init__.py │ │ │ │ ├── config.py │ │ │ │ └── participants.py │ │ │ ├── message/ │ │ │ │ ├── __init__.py │ │ │ │ ├── config.py │ │ │ │ └── message.py │ │ │ └── user/ │ │ │ ├── __init__.py │ │ │ ├── authentication.py │ │ │ ├── config.py │ │ │ ├── contact.py │ │ │ ├── imports.py │ │ │ ├── settings.py │ │ │ ├── user.py │ │ │ └── util.py │ │ ├── requirements.deps │ │ ├── setup.cfg │ │ └── setup.py │ ├── main/ │ │ ├── go.backends/ │ │ │ ├── AttachmentsInterfaces.go │ │ │ ├── CacheInterfaces.go │ │ │ ├── ContactsInterfaces.go │ │ │ ├── CredentialsInterfaces.go │ │ │ ├── DevicesInterfaces.go │ │ │ ├── DiscussionsInterface.go │ │ │ ├── IdentitiesInterface.go │ │ │ ├── KeysInterfaces.go │ │ │ ├── LDAInterfaces.go │ │ │ ├── MessagesInterfaces.go │ │ │ ├── NotificationsInterfaces.go │ │ │ ├── ProvidersInterfaces.go │ │ │ ├── RESTInterfaces.go │ │ │ ├── TagsInterfaces.go │ │ │ ├── UrisInterface.go │ │ │ ├── UsersInterfaces.go │ │ │ ├── backendstest/ │ │ │ │ ├── APIstore.go │ │ │ │ ├── Attachments.go │ │ │ │ ├── Cache.go │ │ │ │ ├── Contacts.go │ │ │ │ ├── Credentials.go │ │ │ │ ├── Devices.go │ │ │ │ ├── Discussions.go │ │ │ │ ├── Identities.go │ │ │ │ ├── Keys.go │ │ │ │ ├── LDA.go │ │ │ │ ├── Messages.go │ │ │ │ ├── Notifications.go │ │ │ │ ├── Providers.go │ │ │ │ ├── Tags.go │ │ │ │ ├── UrisInterface.go │ │ │ │ ├── UserNames.go │ │ │ │ ├── Users.go │ │ │ │ └── testdata.go │ │ │ ├── cache/ │ │ │ │ ├── authentication.go │ │ │ │ ├── cache.go │ │ │ │ ├── devicevalidation.go │ │ │ │ ├── devicevalidation_test.go │ │ │ │ ├── oauthsessions.go │ │ │ │ ├── passwordreset.go │ │ │ │ └── redis.go │ │ │ ├── index/ │ │ │ │ └── elasticsearch/ │ │ │ │ ├── broad_search.go │ │ │ │ ├── contacts.go │ │ │ │ ├── discussions.go │ │ │ │ ├── elasticsearch.go │ │ │ │ ├── messages.go │ │ │ │ └── user_recipients_lookup.go │ │ │ └── store/ │ │ │ ├── cassandra/ │ │ │ │ ├── attachments.go │ │ │ │ ├── cassandra.go │ │ │ │ ├── contacts.go │ │ │ │ ├── contacts_test.go │ │ │ │ ├── credentials.go │ │ │ │ ├── devices.go │ │ │ │ ├── discussions.go │ │ │ │ ├── discussions_test.go │ │ │ │ ├── emails.go │ │ │ │ ├── identities.go │ │ │ │ ├── keys.go │ │ │ │ ├── lookups.go │ │ │ │ ├── messages.go │ │ │ │ ├── notifications.go │ │ │ │ ├── participant_lookup.go │ │ │ │ ├── providers.go │ │ │ │ ├── raw_messages.go │ │ │ │ ├── related.go │ │ │ │ ├── settings.go │ │ │ │ ├── tags.go │ │ │ │ ├── usernames.go │ │ │ │ └── users.go │ │ │ ├── object_store/ │ │ │ │ ├── attachments.go │ │ │ │ ├── minio.go │ │ │ │ ├── objects.go │ │ │ │ └── raw_messages.go │ │ │ └── vault/ │ │ │ ├── credentials.go │ │ │ ├── hvault_interface.go │ │ │ └── vault_client.go │ │ ├── go.main/ │ │ │ ├── caliopen.go │ │ │ ├── contact/ │ │ │ │ ├── contact.go │ │ │ │ ├── contact_test.go │ │ │ │ └── vcard.go │ │ │ ├── facilities/ │ │ │ │ ├── Messaging/ │ │ │ │ │ ├── facility.go │ │ │ │ │ └── facility_test.go │ │ │ │ ├── Notifications/ │ │ │ │ │ ├── batch.go │ │ │ │ │ ├── batch_test.go │ │ │ │ │ ├── email.go │ │ │ │ │ ├── facility.go │ │ │ │ │ ├── queue.go │ │ │ │ │ └── templating.go │ │ │ │ └── REST/ │ │ │ │ ├── RESTfacility.go │ │ │ │ ├── attachment.go │ │ │ │ ├── contacts.go │ │ │ │ ├── contacts_test.go │ │ │ │ ├── devices.go │ │ │ │ ├── devices_test.go │ │ │ │ ├── discussions.go │ │ │ │ ├── draft.go │ │ │ │ ├── identities.go │ │ │ │ ├── keys.go │ │ │ │ ├── message.go │ │ │ │ ├── nats.go │ │ │ │ ├── providers.go │ │ │ │ ├── search.go │ │ │ │ ├── settings.go │ │ │ │ ├── suggest_participants.go │ │ │ │ ├── tags.go │ │ │ │ ├── username.go │ │ │ │ └── users.go │ │ │ ├── helpers/ │ │ │ │ ├── contact.go │ │ │ │ ├── discussion.go │ │ │ │ ├── discussion_test.go │ │ │ │ ├── filesystem.go │ │ │ │ ├── misc.go │ │ │ │ ├── netTest.go │ │ │ │ ├── patch.go │ │ │ │ └── uuid.go │ │ │ ├── messages/ │ │ │ │ └── messages.go │ │ │ ├── pi/ │ │ │ │ ├── identities.go │ │ │ │ └── message.go │ │ │ └── users/ │ │ │ ├── oauth2.go │ │ │ └── password.go │ │ ├── py.main/ │ │ │ ├── CHANGES.rst │ │ │ ├── MANIFEST.in │ │ │ ├── README.rst │ │ │ ├── caliopen_main/ │ │ │ │ ├── __init__.py │ │ │ │ ├── common/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── core/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── base.py │ │ │ │ │ │ ├── pubkey.py │ │ │ │ │ │ └── related.py │ │ │ │ │ ├── errors.py │ │ │ │ │ ├── helpers/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── normalize.py │ │ │ │ │ │ └── strings.py │ │ │ │ │ ├── interfaces/ │ │ │ │ │ │ ├── IO.py │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── parser.py │ │ │ │ │ │ └── storage.py │ │ │ │ │ ├── objects/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── base.py │ │ │ │ │ │ └── tag.py │ │ │ │ │ ├── parameters/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── pubkey.py │ │ │ │ │ │ ├── tag.py │ │ │ │ │ │ └── types.py │ │ │ │ │ └── store/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── pubkey.py │ │ │ │ │ └── tag.py │ │ │ │ ├── contact/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── core.py │ │ │ │ │ ├── objects/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── contact.py │ │ │ │ │ │ ├── email.py │ │ │ │ │ │ ├── identity.py │ │ │ │ │ │ ├── im.py │ │ │ │ │ │ ├── organization.py │ │ │ │ │ │ ├── phone.py │ │ │ │ │ │ └── postal_address.py │ │ │ │ │ ├── parameters.py │ │ │ │ │ ├── parsers/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── vcard.py │ │ │ │ │ ├── returns.py │ │ │ │ │ └── store/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── contact.py │ │ │ │ │ └── contact_index.py │ │ │ │ ├── device/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── core.py │ │ │ │ │ ├── parameters.py │ │ │ │ │ └── store.py │ │ │ │ ├── discussion/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── core/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── discussion.py │ │ │ │ │ ├── objects/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── discussion.py │ │ │ │ │ ├── parameters/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── discussion.py │ │ │ │ │ └── store/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── discussion_index.py │ │ │ │ │ └── discussion_lookup.py │ │ │ │ ├── message/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── core/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── external_references.py │ │ │ │ │ │ └── raw.py │ │ │ │ │ ├── objects/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── attachment.py │ │ │ │ │ │ ├── external_references.py │ │ │ │ │ │ └── message.py │ │ │ │ │ ├── parameters/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── attachment.py │ │ │ │ │ │ ├── draft.py │ │ │ │ │ │ ├── external_references.py │ │ │ │ │ │ └── message.py │ │ │ │ │ ├── parsers/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── mail.py │ │ │ │ │ │ ├── mastodon.py │ │ │ │ │ │ └── twitter.py │ │ │ │ │ └── store/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── attachment.py │ │ │ │ │ ├── attachment_index.py │ │ │ │ │ ├── external_references.py │ │ │ │ │ ├── external_references_index.py │ │ │ │ │ ├── message.py │ │ │ │ │ ├── message_index.py │ │ │ │ │ └── raw.py │ │ │ │ ├── notification/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── core.py │ │ │ │ │ └── store.py │ │ │ │ ├── participant/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── core/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── participant.py │ │ │ │ │ ├── objects/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── participant.py │ │ │ │ │ ├── parameters/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── participant.py │ │ │ │ │ └── store/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── participant.py │ │ │ │ │ └── participant_index.py │ │ │ │ ├── pi/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── objects.py │ │ │ │ │ └── parameters.py │ │ │ │ ├── protocol/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── core/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── provider.py │ │ │ │ │ └── store/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── provider.py │ │ │ │ ├── tests/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── fixtures/ │ │ │ │ │ │ ├── mail/ │ │ │ │ │ │ │ ├── pgp_crypted_1.eml │ │ │ │ │ │ │ └── pgp_signed_1.eml │ │ │ │ │ │ └── vcard/ │ │ │ │ │ │ ├── multi.vcf │ │ │ │ │ │ ├── rfc2425-1.vcard │ │ │ │ │ │ ├── rfc2425-2.vcard │ │ │ │ │ │ ├── rfc2425-3.vcard │ │ │ │ │ │ ├── rfc2426-1.vcard │ │ │ │ │ │ ├── rfc2426-2.vcard │ │ │ │ │ │ ├── rfc2426-3.vcard │ │ │ │ │ │ ├── rfc2426-4.vcard │ │ │ │ │ │ ├── rfc2426-5.vcard │ │ │ │ │ │ ├── vcard1.vcf │ │ │ │ │ │ ├── vcard2.vcf │ │ │ │ │ │ └── vcard3.vcf │ │ │ │ │ └── parsers/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── test_email.py │ │ │ │ │ ├── test_mail.py │ │ │ │ │ └── test_vcard.py │ │ │ │ └── user/ │ │ │ │ ├── __init__.py │ │ │ │ ├── core/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── identity.py │ │ │ │ │ ├── setups.py │ │ │ │ │ └── user.py │ │ │ │ ├── helpers/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── mergePatch.py │ │ │ │ │ └── validators.py │ │ │ │ ├── objects/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── device.py │ │ │ │ │ ├── identity.py │ │ │ │ │ ├── settings.py │ │ │ │ │ └── tag.py │ │ │ │ ├── parameters/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── identity.py │ │ │ │ │ ├── settings.py │ │ │ │ │ ├── tag.py │ │ │ │ │ └── user.py │ │ │ │ ├── returns/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── user.py │ │ │ │ └── store/ │ │ │ │ ├── __init__.py │ │ │ │ ├── identity.py │ │ │ │ ├── tag.py │ │ │ │ ├── user.py │ │ │ │ └── user_index.py │ │ │ ├── requirements.deps │ │ │ ├── setup.cfg │ │ │ └── setup.py │ │ └── py.storage/ │ │ ├── CHANGES.rst │ │ ├── MANIFEST.in │ │ ├── README.rst │ │ ├── caliopen_storage/ │ │ │ ├── __init__.py │ │ │ ├── config.py │ │ │ ├── core/ │ │ │ │ ├── __init__.py │ │ │ │ ├── base.py │ │ │ │ ├── mixin.py │ │ │ │ └── registry.py │ │ │ ├── exception.py │ │ │ ├── helpers/ │ │ │ │ ├── __init__.py │ │ │ │ ├── connection.py │ │ │ │ └── json.py │ │ │ ├── parameters.py │ │ │ ├── returns.py │ │ │ └── store/ │ │ │ ├── __init__.py │ │ │ ├── mixin.py │ │ │ └── model.py │ │ ├── setup.cfg │ │ └── setup.py │ ├── protocols/ │ │ ├── go.imap/ │ │ │ ├── cmd/ │ │ │ │ ├── imapctl/ │ │ │ │ │ ├── cli_cmds/ │ │ │ │ │ │ ├── addremote.go │ │ │ │ │ │ ├── fullfetch.go │ │ │ │ │ │ ├── root.go │ │ │ │ │ │ └── syncremote.go │ │ │ │ │ └── main.go │ │ │ │ └── imapworker/ │ │ │ │ ├── cli_cmds/ │ │ │ │ │ ├── root.go │ │ │ │ │ └── start.go │ │ │ │ └── main.go │ │ │ ├── config.go │ │ │ ├── fetcher.go │ │ │ ├── imap.go │ │ │ ├── imap_test.go │ │ │ ├── lda.go │ │ │ ├── sender.go │ │ │ ├── sender_test.go │ │ │ ├── worker.go │ │ │ └── worker_test.go │ │ ├── go.mastodon/ │ │ │ ├── account.go │ │ │ ├── cmd/ │ │ │ │ └── mastodonworker/ │ │ │ │ ├── cli_cmds/ │ │ │ │ │ ├── root.go │ │ │ │ │ └── start.go │ │ │ │ └── main.go │ │ │ ├── messaging.go │ │ │ └── worker.go │ │ ├── go.smtp/ │ │ │ ├── cmd/ │ │ │ │ └── caliopen_lmtpd/ │ │ │ │ ├── cli_cmds/ │ │ │ │ │ ├── root.go │ │ │ │ │ └── serve.go │ │ │ │ └── main.go │ │ │ ├── config.go │ │ │ ├── envelope.go │ │ │ ├── lda.go │ │ │ ├── lmtpd.go │ │ │ ├── oauth.go │ │ │ ├── protocol.go │ │ │ ├── receiver.go │ │ │ ├── server.go │ │ │ └── submitter.go │ │ └── go.twitter/ │ │ ├── account.go │ │ ├── account_test.go │ │ ├── cmd/ │ │ │ └── twitterworker/ │ │ │ ├── cli_cmds/ │ │ │ │ ├── root.go │ │ │ │ └── start.go │ │ │ └── main.go │ │ ├── messaging.go │ │ ├── messaging_test.go │ │ ├── worker.go │ │ └── worker_test.go │ ├── tools/ │ │ ├── go.CLI/ │ │ │ └── cmd/ │ │ │ └── gocaliopen/ │ │ │ ├── cli_cmds/ │ │ │ │ ├── changeIdentitiyEmailProtocol.go │ │ │ │ ├── changeUserIdentitiesCredentialsKeys.go │ │ │ │ ├── fixMissingParticipants.go │ │ │ │ ├── identitiesMigration.go │ │ │ │ └── root.go │ │ │ └── main.go │ │ ├── py.CLI/ │ │ │ ├── CHANGES.rst │ │ │ ├── README.rst │ │ │ ├── caliopen_cli/ │ │ │ │ ├── __init__.py │ │ │ │ ├── cli.py │ │ │ │ ├── commands/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── compute.py │ │ │ │ │ ├── copy_model.py │ │ │ │ │ ├── create_user.py │ │ │ │ │ ├── dump_indexes_mappings.py │ │ │ │ │ ├── dump_model.py │ │ │ │ │ ├── import_email.py │ │ │ │ │ ├── import_vcard.py │ │ │ │ │ ├── inject_email.py │ │ │ │ │ ├── migrate_index.py │ │ │ │ │ ├── reserved_names.py │ │ │ │ │ ├── resync_index.py │ │ │ │ │ ├── resync_shard_index.py │ │ │ │ │ ├── setup.py │ │ │ │ │ ├── setup_notifications_ttls.py │ │ │ │ │ ├── setup_storage.py │ │ │ │ │ └── shell.py │ │ │ │ └── utils/ │ │ │ │ ├── __init__.py │ │ │ │ └── user_token.py │ │ │ ├── requirements.deps │ │ │ ├── setup.cfg │ │ │ └── setup.py │ │ ├── py.ML/ │ │ │ ├── CHANGES.rst │ │ │ ├── README.rst │ │ │ ├── caliopen_climl/ │ │ │ │ ├── __init__.py │ │ │ │ └── cli.py │ │ │ ├── requirements.deps │ │ │ ├── setup.cfg │ │ │ └── setup.py │ │ ├── py.doc/ │ │ │ ├── CHANGES.rst │ │ │ ├── README.rst │ │ │ ├── caliopen_api_doc/ │ │ │ │ ├── __init__.py │ │ │ │ ├── config.py │ │ │ │ └── swagger-ui/ │ │ │ │ ├── css/ │ │ │ │ │ ├── print.css │ │ │ │ │ ├── reset.css │ │ │ │ │ ├── screen.css │ │ │ │ │ ├── style.css │ │ │ │ │ └── typography.css │ │ │ │ ├── index.html │ │ │ │ ├── lang/ │ │ │ │ │ ├── ca.js │ │ │ │ │ ├── el.js │ │ │ │ │ ├── en.js │ │ │ │ │ ├── es.js │ │ │ │ │ ├── fr.js │ │ │ │ │ ├── geo.js │ │ │ │ │ ├── it.js │ │ │ │ │ ├── ja.js │ │ │ │ │ ├── ko-kr.js │ │ │ │ │ ├── pl.js │ │ │ │ │ ├── pt.js │ │ │ │ │ ├── ru.js │ │ │ │ │ ├── tr.js │ │ │ │ │ ├── translator.js │ │ │ │ │ └── zh-cn.js │ │ │ │ ├── lib/ │ │ │ │ │ ├── backbone-min.js │ │ │ │ │ ├── es5-shim.js │ │ │ │ │ ├── handlebars-4.0.5.js │ │ │ │ │ ├── highlight.9.1.0.pack.js │ │ │ │ │ ├── highlight.9.1.0.pack_extended.js │ │ │ │ │ ├── marked.js │ │ │ │ │ ├── object-assign-pollyfill.js │ │ │ │ │ └── swagger-oauth.js │ │ │ │ ├── o2c.html │ │ │ │ └── swagger-ui.js │ │ │ ├── setup.cfg │ │ │ └── setup.py │ │ └── py.migrate/ │ │ ├── CHANGES.rst │ │ ├── README.rst │ │ ├── caliopen_migrate/ │ │ │ ├── __init__.py │ │ │ ├── cli.py │ │ │ ├── shards.py │ │ │ ├── user.py │ │ │ └── welcome_message.py │ │ ├── setup.cfg │ │ └── setup.py │ └── workers/ │ └── go.remoteIDs/ │ ├── cmd/ │ │ └── idpoller/ │ │ ├── cli_cmds/ │ │ │ ├── root.go │ │ │ └── start.go │ │ └── main.go │ ├── identities.go │ ├── identities_test.go │ ├── idpoller.go │ ├── idpollertest/ │ │ └── testsreplies.go │ ├── jobs.go │ ├── jobs_test.go │ ├── messaging.go │ ├── messaging_test.go │ ├── scheduler.go │ └── scheduler_test.go └── frontend/ ├── maintenance/ │ ├── Dockerfile │ ├── README.md │ ├── config/ │ │ └── nginx-config-maintenance.conf │ ├── package.json │ └── src/ │ ├── assets/ │ │ ├── css/ │ │ │ └── style.css │ │ ├── js/ │ │ │ └── init.js │ │ └── scss/ │ │ ├── params/ │ │ │ ├── _fonts.scss │ │ │ ├── _grid.scss │ │ │ ├── _images.scss │ │ │ └── _variables.scss │ │ └── style.scss │ └── index.html ├── not_found/ │ ├── 404.html │ └── assets/ │ └── app.client.e4ae8e333ea54a53ba14.css └── web_application/ ├── .babelrc ├── .dockerignore ├── .eslintrc.json ├── .gitignore ├── .prettierignore ├── .prettierrc.js ├── .stylelintrc.yml ├── CONTRIBUTING.md ├── Dockerfile ├── Dockerfile.caliopen-node ├── README.md ├── __mocks__/ │ ├── fileMock.js │ └── styleMock.js ├── bin/ │ ├── dev-server │ └── server ├── config/ │ ├── client.default.js │ ├── server.default.js │ └── server.env-var.js ├── lingui.config.ts ├── locale/ │ ├── de/ │ │ └── messages.json │ ├── en/ │ │ └── messages.json │ ├── es/ │ │ └── messages.json │ └── fr/ │ └── messages.json ├── package.json ├── postcss.config.js ├── public/ │ └── privacy-policy.html ├── server/ │ ├── api/ │ │ ├── index.js │ │ └── lib/ │ │ ├── api.ts │ │ └── sub-request-manager.js │ ├── app.ts │ ├── assets/ │ │ └── index.js │ ├── auth/ │ │ ├── index.js │ │ ├── lib/ │ │ │ ├── cookie.ts │ │ │ ├── redirect.ts │ │ │ └── seal.ts │ │ ├── middlewares/ │ │ │ ├── decode-cookie.ts │ │ │ └── index.ts │ │ └── router/ │ │ ├── index.js │ │ ├── signin.js │ │ └── signup.ts │ ├── config/ │ │ └── index.js │ ├── error/ │ │ ├── components/ │ │ │ └── Error/ │ │ │ └── index.jsx │ │ ├── consts.ts │ │ ├── index.js │ │ └── middlewares/ │ │ └── catch-error.js │ ├── express-react/ │ │ ├── create-engine.js │ │ ├── index.js │ │ └── view.js │ ├── index.js │ ├── logger/ │ │ ├── getLogger.js │ │ ├── index.js │ │ └── middlewares/ │ │ └── httpLog.js │ └── ssr/ │ ├── components/ │ │ └── Bootstrap.tsx │ ├── index.js │ └── ssr-middleware.ts ├── src/ │ ├── App.tsx │ ├── app.scss │ ├── components/ │ │ ├── ActionBar/ │ │ │ ├── components/ │ │ │ │ ├── ActionBarButton/ │ │ │ │ │ └── index.tsx │ │ │ │ └── ActionBarWrapper/ │ │ │ │ ├── index.tsx │ │ │ │ └── styles.scss │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── AdvancedSelectFieldGroup/ │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── AppLoader/ │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── Badge/ │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── BlockList/ │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── Brand/ │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── Button/ │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── Callout/ │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── Checkbox/ │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── CheckboxFieldGroup/ │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── Confirm/ │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── DatePickerGroup/ │ │ │ ├── index.jsx │ │ │ ├── presenter.jsx │ │ │ └── style.scss │ │ ├── DefList/ │ │ │ ├── index.jsx │ │ │ ├── index.spec.jsx │ │ │ └── style.scss │ │ ├── Dropdown/ │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ ├── services/ │ │ │ │ ├── getDropdownStyle.spec.ts │ │ │ │ └── getDropdownStyle.ts │ │ │ └── style.scss │ │ ├── DropdownMenu/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── FieldErrors/ │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── FieldGroup/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── Fieldset/ │ │ │ ├── components/ │ │ │ │ └── Legend/ │ │ │ │ └── index.jsx │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── FileSize/ │ │ │ ├── index.spec.tsx │ │ │ └── index.tsx │ │ ├── FormGrid/ │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── Icon/ │ │ │ ├── __snapshots__/ │ │ │ │ └── index.spec.tsx.snap │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── InfiniteScroll/ │ │ │ └── index.jsx │ │ ├── InputFile/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── InputFileGroup/ │ │ │ ├── components/ │ │ │ │ └── File/ │ │ │ │ └── index.jsx │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── InputText/ │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── Label/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── Link/ │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── MenuBar/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── MessageDate/ │ │ │ ├── index.js │ │ │ ├── presenter.jsx │ │ │ └── style.scss │ │ ├── Modal/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── NavList/ │ │ │ ├── components/ │ │ │ │ └── NavItem.tsx │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── PageTitle/ │ │ │ ├── index.jsx │ │ │ └── presenter.jsx │ │ ├── ParticipantsIconLetter/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── PasswordStrength/ │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── PiBar/ │ │ │ ├── index.js │ │ │ ├── presenter.jsx │ │ │ └── style.scss │ │ ├── PlaceholderBlock/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── PlaceholderList/ │ │ │ ├── index.scss │ │ │ └── index.tsx │ │ ├── RadioFieldGroup/ │ │ │ ├── index.jsx │ │ │ ├── index.spec.jsx │ │ │ └── style.scss │ │ ├── RawButton/ │ │ │ └── index.tsx │ │ ├── Section/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── SelectFieldGroup/ │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── SidebarLayout/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── Spinner/ │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── Subtitle/ │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── Switch/ │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── TextBlock/ │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── TextFieldGroup/ │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── TextList/ │ │ │ ├── components/ │ │ │ │ └── TextItem.tsx │ │ │ ├── index.spec.jsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── Textarea/ │ │ │ ├── index.jsx │ │ │ ├── index.spec.jsx │ │ │ └── style.scss │ │ ├── TextareaFieldGroup/ │ │ │ ├── index.jsx │ │ │ ├── index.spec.jsx │ │ │ └── style.scss │ │ ├── Title/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── VerticalMenu/ │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ └── index.js │ ├── global.d.ts │ ├── hooks/ │ │ └── forwardedRef.ts │ ├── image.d.ts │ ├── index.jsx │ ├── layouts/ │ │ ├── AboutPage/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── AuthPage/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── ErrorBoundary/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── Page/ │ │ │ ├── components/ │ │ │ │ ├── Footer/ │ │ │ │ │ ├── footer.scss │ │ │ │ │ └── index.jsx │ │ │ │ ├── HorizontalScroll/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── InstallButton/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── Navbar/ │ │ │ │ │ └── components/ │ │ │ │ │ ├── ApplicationTab/ │ │ │ │ │ │ ├── application-tab.scss │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── ContactAssociationTab/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── ContactTab/ │ │ │ │ │ │ └── index.tsx │ │ │ │ │ ├── DiscussionTab/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── ItemButton/ │ │ │ │ │ │ ├── index.jsx │ │ │ │ │ │ └── style.scss │ │ │ │ │ ├── ItemLink/ │ │ │ │ │ │ ├── index.jsx │ │ │ │ │ │ └── style.scss │ │ │ │ │ ├── NavbarItem/ │ │ │ │ │ │ ├── index.jsx │ │ │ │ │ │ └── style.scss │ │ │ │ │ ├── SearchTab/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── Tab/ │ │ │ │ │ │ ├── index.jsx │ │ │ │ │ │ └── style.scss │ │ │ │ │ └── index.js │ │ │ │ ├── Navigation/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ ├── presenter.jsx │ │ │ │ │ ├── style.scss │ │ │ │ │ └── withTabs.jsx │ │ │ │ ├── NotificationCenter/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ └── TakeATour/ │ │ │ │ ├── components/ │ │ │ │ │ ├── Guide/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── Tour/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── TourPortal/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── index.js │ │ │ │ ├── presenter.jsx │ │ │ │ └── style.scss │ │ │ ├── header.scss │ │ │ ├── index.jsx │ │ │ ├── navbar.scss │ │ │ └── style.scss │ │ ├── PageContainer/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── SearchResults/ │ │ │ ├── index.js │ │ │ ├── presenter.jsx │ │ │ └── style.scss │ │ ├── Settings/ │ │ │ ├── index.js │ │ │ ├── presenter.jsx │ │ │ └── style.scss │ │ └── User/ │ │ ├── index.js │ │ ├── presenter.jsx │ │ └── styles.scss │ ├── modules/ │ │ ├── a11y/ │ │ │ ├── index.js │ │ │ └── services/ │ │ │ └── tabIndexes.js │ │ ├── avatar/ │ │ │ ├── components/ │ │ │ │ ├── AuthorAvatarLetter/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── index.spec.jsx │ │ │ │ ├── AvatarLetter/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ ├── services/ │ │ │ │ │ │ └── stylesheet-helper/ │ │ │ │ │ │ ├── index.js │ │ │ │ │ │ └── index.spec.js │ │ │ │ │ └── style.scss │ │ │ │ ├── AvatarLetterWrapper/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ └── ContactAvatarLetter/ │ │ │ │ ├── index.jsx │ │ │ │ └── index.spec.jsx │ │ │ ├── index.js │ │ │ └── services/ │ │ │ └── stylesheet-helper/ │ │ │ ├── index.js │ │ │ └── index.spec.js │ │ ├── contact/ │ │ │ ├── actions/ │ │ │ │ ├── deleteContacts.js │ │ │ │ ├── getContact.ts │ │ │ │ ├── loadMoreContacts.ts │ │ │ │ └── updateContact.js │ │ │ ├── components/ │ │ │ │ ├── ContactList/ │ │ │ │ │ ├── components/ │ │ │ │ │ │ ├── ContactItem/ │ │ │ │ │ │ │ ├── index.tsx │ │ │ │ │ │ │ └── style.scss │ │ │ │ │ │ └── ContactItemPlaceholder.tsx │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ └── WithContacts/ │ │ │ │ ├── index.js │ │ │ │ └── presenter.jsx │ │ │ ├── consts.ts │ │ │ ├── hoc/ │ │ │ │ └── withContacts.jsx │ │ │ ├── hooks/ │ │ │ │ └── useContacts.ts │ │ │ ├── index.ts │ │ │ ├── query.ts │ │ │ ├── selectors/ │ │ │ │ └── contactSelector.ts │ │ │ ├── services/ │ │ │ │ ├── addAddressToContact.js │ │ │ │ ├── form.ts │ │ │ │ ├── format.spec.ts │ │ │ │ ├── format.ts │ │ │ │ └── identityTypes.js │ │ │ ├── store/ │ │ │ │ ├── index.ts │ │ │ │ ├── reducer.ts │ │ │ │ └── selectors.ts │ │ │ └── types.d.ts │ │ ├── control/ │ │ │ ├── components/ │ │ │ │ ├── ComposeButton/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── ComposeContactButton/ │ │ │ │ │ ├── index.js │ │ │ │ │ └── presenter.jsx │ │ │ │ ├── CreateContactButton/ │ │ │ │ │ └── index.jsx │ │ │ │ └── PageActions/ │ │ │ │ ├── action-btns.scss │ │ │ │ ├── index.jsx │ │ │ │ └── style.scss │ │ │ └── index.js │ │ ├── device/ │ │ │ ├── actions/ │ │ │ │ ├── requestDevice.js │ │ │ │ ├── requestDevices.js │ │ │ │ ├── revokeDevice.js │ │ │ │ ├── saveDevice.js │ │ │ │ └── verifyDevice.js │ │ │ ├── hooks/ │ │ │ │ ├── useDevice.ts │ │ │ │ └── useDevices.ts │ │ │ ├── index.ts │ │ │ ├── selectors.ts │ │ │ ├── services/ │ │ │ │ ├── clientDevice.ts │ │ │ │ ├── ecdsa/ │ │ │ │ │ └── index.ts │ │ │ │ ├── signature/ │ │ │ │ │ ├── index.spec.ts │ │ │ │ │ └── index.ts │ │ │ │ ├── status.js │ │ │ │ └── storage/ │ │ │ │ └── index.ts │ │ │ └── types.d.ts │ │ ├── discussion/ │ │ │ ├── actions/ │ │ │ │ └── requestDiscussionIdForParticipants.js │ │ │ ├── hooks/ │ │ │ │ └── useDiscussion.ts │ │ │ ├── index.js │ │ │ ├── query.ts │ │ │ └── selectors/ │ │ │ ├── discussionDraftSelector.js │ │ │ ├── discussionIdSelector.js │ │ │ ├── discussionSelector.js │ │ │ └── discussionsSelector.ts │ │ ├── draftIdentity/ │ │ │ ├── hooks/ │ │ │ │ └── useAvailableIdentities.ts │ │ │ └── index.ts │ │ ├── draftMessage/ │ │ │ ├── actions/ │ │ │ │ ├── consolidateParticipants.js │ │ │ │ ├── getDefaultIdentity.js │ │ │ │ ├── getDefaultIdentity.spec.js │ │ │ │ ├── reply.ts │ │ │ │ ├── requestDraft.ts │ │ │ │ ├── requestParticipantSuggestions.spec.ts │ │ │ │ ├── requestParticipantSuggestions.ts │ │ │ │ ├── saveDraft.ts │ │ │ │ └── sendDraft.ts │ │ │ ├── components/ │ │ │ │ ├── AttachmentManager/ │ │ │ │ │ ├── index.js │ │ │ │ │ ├── presenter.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── Recipient/ │ │ │ │ │ ├── index.js │ │ │ │ │ ├── presenter.jsx │ │ │ │ │ └── presenter.spec.jsx │ │ │ │ ├── RecipientList.scss │ │ │ │ ├── RecipientList.spec.tsx │ │ │ │ ├── RecipientList.tsx │ │ │ │ └── RecipientSelector.tsx │ │ │ ├── index.js │ │ │ ├── models.ts │ │ │ ├── selectors/ │ │ │ │ └── draft.ts │ │ │ ├── services/ │ │ │ │ ├── calcSyncDraft.spec.ts │ │ │ │ ├── calcSyncDraft.ts │ │ │ │ ├── changeAuthorInParticipants.js │ │ │ │ ├── changeAuthorInParticipants.spec.js │ │ │ │ ├── filterIdentities.js │ │ │ │ ├── filterIdentities.spec.js │ │ │ │ ├── getIdentityProtocol.js │ │ │ │ ├── isValidRecipient.js │ │ │ │ ├── isValidRecipient.spec.js │ │ │ │ └── validate.ts │ │ │ └── types.d.ts │ │ ├── encryption/ │ │ │ ├── actions/ │ │ │ │ ├── decryptMessage.js │ │ │ │ ├── encryptMessage.js │ │ │ │ ├── fetchRemoteKeys.js │ │ │ │ └── getRecipientKeys.js │ │ │ ├── components/ │ │ │ │ ├── AskPassphraseForm/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── CheckDecryption/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ └── LockedMessage/ │ │ │ │ ├── index.jsx │ │ │ │ └── style.scss │ │ │ ├── index.js │ │ │ ├── selectors/ │ │ │ │ ├── message.ts │ │ │ │ └── publicKey.js │ │ │ └── services/ │ │ │ └── keyring/ │ │ │ └── remoteKeys.js │ │ ├── file/ │ │ │ ├── actions/ │ │ │ │ ├── deleteDraftAttachment.js │ │ │ │ └── uploadDraftAttachments.js │ │ │ ├── index.js │ │ │ └── services/ │ │ │ ├── index.js │ │ │ └── uploadFileAsFormField.js │ │ ├── form/ │ │ │ ├── components/ │ │ │ │ └── FormikPersist.tsx │ │ │ └── services/ │ │ │ └── validators.ts │ │ ├── i18n/ │ │ │ ├── components/ │ │ │ │ └── I18nLoader.tsx │ │ │ ├── index.js │ │ │ └── services/ │ │ │ ├── getBestLocale.js │ │ │ ├── getLanguage.js │ │ │ └── getUserLocales.js │ │ ├── identity/ │ │ │ ├── actions/ │ │ │ │ ├── getIdentities.js │ │ │ │ ├── getLocalIdentities.js │ │ │ │ └── getRemoteIdentities.js │ │ │ ├── components/ │ │ │ │ └── WithIdentities/ │ │ │ │ ├── index.js │ │ │ │ └── presenter.jsx │ │ │ ├── hoc/ │ │ │ │ └── withIdentities.jsx │ │ │ ├── hooks/ │ │ │ │ └── useIdentities.ts │ │ │ ├── index.js │ │ │ ├── selectors/ │ │ │ │ └── identitiesSelector.js │ │ │ ├── services/ │ │ │ │ └── identityToParticipant.js │ │ │ └── types.d.ts │ │ ├── message/ │ │ │ ├── actions/ │ │ │ │ ├── createMessage.js │ │ │ │ ├── deleteMessage.js │ │ │ │ ├── fetchMessages.js │ │ │ │ ├── getDraft.spec.ts │ │ │ │ ├── getDraft.ts │ │ │ │ ├── getLastMessage.js │ │ │ │ ├── getLastMessage.spec.js │ │ │ │ ├── getMessage.js │ │ │ │ ├── getMessages.js │ │ │ │ ├── getParentMessage.js │ │ │ │ ├── requestDiscussion.js │ │ │ │ ├── requestMessages.js │ │ │ │ └── setMessageRead.js │ │ │ ├── components/ │ │ │ │ └── ParticipantLabel/ │ │ │ │ ├── index.js │ │ │ │ └── presenter.jsx │ │ │ ├── hooks/ │ │ │ │ └── useMessage.ts │ │ │ ├── index.js │ │ │ ├── models/ │ │ │ │ ├── Message.ts │ │ │ │ └── Participant.ts │ │ │ ├── query.ts │ │ │ ├── selectors/ │ │ │ │ └── messageSelector.js │ │ │ ├── services/ │ │ │ │ ├── findUserParticipant.js │ │ │ │ ├── findUserParticipant.spec.js │ │ │ │ ├── getLastMessageFromArray.js │ │ │ │ ├── isUserParticipant.js │ │ │ │ ├── isUserParticipant.spec.js │ │ │ │ └── sortMessages.js │ │ │ └── types.d.ts │ │ ├── notification/ │ │ │ ├── components/ │ │ │ │ ├── MessageNotificationHandler/ │ │ │ │ │ ├── index.js │ │ │ │ │ └── presenter.jsx │ │ │ │ └── NotificationProvider/ │ │ │ │ ├── index.js │ │ │ │ └── presenter.jsx │ │ │ ├── index.js │ │ │ ├── selectors/ │ │ │ │ └── messageNotificationsSelector.js │ │ │ └── services/ │ │ │ └── notification.worker.js │ │ ├── pi/ │ │ │ ├── components/ │ │ │ │ ├── BackgroundImage/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ ├── o-background-image.scss │ │ │ │ │ └── style.scss │ │ │ │ ├── MultidimensionalPi/ │ │ │ │ │ ├── index.js │ │ │ │ │ ├── presenter.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── PiGraph/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── PiScore/ │ │ │ │ │ └── index.tsx │ │ │ │ └── Ratings/ │ │ │ │ ├── index.jsx │ │ │ │ └── style.scss │ │ │ ├── index.jsx │ │ │ ├── services/ │ │ │ │ ├── pi/ │ │ │ │ │ ├── index.spec.ts │ │ │ │ │ └── index.ts │ │ │ │ └── svg/ │ │ │ │ ├── index.js │ │ │ │ └── index.spec.jsx │ │ │ └── types.d.ts │ │ ├── publicKey/ │ │ │ ├── actions/ │ │ │ │ ├── saveUserPublicKeyAction.js │ │ │ │ └── updatePublicKey.js │ │ │ └── index.js │ │ ├── pwa/ │ │ │ ├── components/ │ │ │ │ ├── InstallPromptConsumer.tsx │ │ │ │ └── InstallPromptProvider.tsx │ │ │ ├── contexts/ │ │ │ │ └── InstallPromptContext.ts │ │ │ └── index.js │ │ ├── remoteIdentity/ │ │ │ ├── actions/ │ │ │ │ ├── createIdentity.js │ │ │ │ ├── deleteIdentity.js │ │ │ │ └── updateIdentity.js │ │ │ ├── components/ │ │ │ │ ├── ProviderIcon/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ └── WithProviders/ │ │ │ │ ├── index.js │ │ │ │ └── presenter.jsx │ │ │ ├── hoc/ │ │ │ │ ├── withAuthorize.jsx │ │ │ │ ├── withAuthorizePopup.jsx │ │ │ │ └── withProviders.jsx │ │ │ ├── index.js │ │ │ ├── model/ │ │ │ │ └── Identity.js │ │ │ ├── selectors/ │ │ │ │ ├── identitiesSelector.js │ │ │ │ ├── identitySelector.js │ │ │ │ └── identityStateSelector.js │ │ │ └── services/ │ │ │ └── getProvider.js │ │ ├── routing/ │ │ │ ├── components/ │ │ │ │ ├── AuthenticatedLayout/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── withAuthenticatedProps.jsx │ │ │ │ ├── RoutingConsumer.jsx │ │ │ │ ├── RoutingProvider.jsx │ │ │ │ └── SwitchWithRoutes/ │ │ │ │ ├── index.jsx │ │ │ │ └── index.spec.jsx │ │ │ ├── contexts/ │ │ │ │ └── RoutingContext.js │ │ │ ├── hoc/ │ │ │ │ ├── withPush.jsx │ │ │ │ ├── withReplace.jsx │ │ │ │ ├── withRouteParams.jsx │ │ │ │ └── withSearchParams.jsx │ │ │ ├── hooks/ │ │ │ │ └── useSearchParams.ts │ │ │ ├── index.js │ │ │ └── services/ │ │ │ ├── findTabbableRouteConfig.js │ │ │ ├── flattenRouteConfig.js │ │ │ ├── flattenRouteConfig.spec.jsx │ │ │ ├── getRouterHistory.js │ │ │ ├── getSearchParams.spec.ts │ │ │ ├── getSearchParams.ts │ │ │ ├── signout.js │ │ │ └── url/ │ │ │ ├── CrappyURLSearchParams.js │ │ │ ├── CrappyURLSearchParams.spec.js │ │ │ ├── QueryStringSerializer.js │ │ │ ├── buildURL.js │ │ │ └── index.js │ │ ├── scroll/ │ │ │ ├── components/ │ │ │ │ └── ScrollDetector/ │ │ │ │ ├── index.js │ │ │ │ └── index.js.w_detection │ │ │ ├── hoc/ │ │ │ │ ├── withScrollManager.jsx │ │ │ │ └── withScrollTarget.jsx │ │ │ ├── hooks/ │ │ │ │ └── useScrollToMe.ts │ │ │ ├── index.js │ │ │ ├── services/ │ │ │ │ ├── getTop.js │ │ │ │ ├── getViewPortTop.js │ │ │ │ └── scrollTop.js │ │ │ └── vendors/ │ │ │ └── scroll-doc.js │ │ ├── search/ │ │ │ ├── components/ │ │ │ │ └── SearchField/ │ │ │ │ ├── index.js │ │ │ │ ├── presenter.jsx │ │ │ │ └── style.scss │ │ │ └── index.js │ │ ├── settings/ │ │ │ ├── components/ │ │ │ │ └── WithSettings/ │ │ │ │ ├── index.js │ │ │ │ ├── presenter.jsx │ │ │ │ └── presenter.spec.jsx │ │ │ ├── hoc/ │ │ │ │ └── withSettings.js │ │ │ ├── hooks/ │ │ │ │ └── useSettings.ts │ │ │ ├── index.js │ │ │ ├── selectors/ │ │ │ │ └── settings.ts │ │ │ ├── services/ │ │ │ │ └── getDefaultSettings/ │ │ │ │ └── index.js │ │ │ └── types.d.ts │ │ ├── tab/ │ │ │ ├── components/ │ │ │ │ ├── TabConsumer/ │ │ │ │ │ └── index.jsx │ │ │ │ └── TabProvider/ │ │ │ │ └── index.jsx │ │ │ ├── contexts/ │ │ │ │ └── TabContext.js │ │ │ ├── hoc/ │ │ │ │ ├── withCloseTab.jsx │ │ │ │ ├── withCurrentTab.jsx │ │ │ │ └── withUpdateTab.jsx │ │ │ ├── hooks/ │ │ │ │ ├── useCloseTab.ts │ │ │ │ └── useCurentTab.ts │ │ │ ├── index.js │ │ │ ├── model/ │ │ │ │ └── Tab.js │ │ │ └── services/ │ │ │ └── getTabUrl.js │ │ ├── tags/ │ │ │ ├── components/ │ │ │ │ ├── ManageEntityTags.spec.tsx │ │ │ │ ├── ManageEntityTags.tsx │ │ │ │ ├── TagFieldGroup/ │ │ │ │ │ ├── index.spec.jsx │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── TagItem/ │ │ │ │ │ ├── __snapshots__/ │ │ │ │ │ │ └── index.spec.tsx.snap │ │ │ │ │ ├── index.spec.tsx │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ └── TagsForm/ │ │ │ │ ├── index.spec.tsx │ │ │ │ ├── index.tsx │ │ │ │ └── style.scss │ │ │ ├── hoc/ │ │ │ │ └── withTags/ │ │ │ │ └── index.jsx │ │ │ ├── hooks/ │ │ │ │ └── useTags.ts │ │ │ ├── index.ts │ │ │ ├── query.ts │ │ │ ├── services/ │ │ │ │ ├── getTagLabel/ │ │ │ │ │ ├── index.spec.ts │ │ │ │ │ └── index.ts │ │ │ │ ├── getTagNamesInCommon.spec.ts │ │ │ │ ├── getTagNamesInCommon.ts │ │ │ │ ├── searchTags.spec.ts │ │ │ │ └── searchTags.ts │ │ │ ├── store/ │ │ │ │ ├── index.ts │ │ │ │ ├── reducer.ts │ │ │ │ └── selectors.ts │ │ │ └── types.d.ts │ │ ├── user/ │ │ │ ├── actions/ │ │ │ │ └── getUser.js │ │ │ ├── components/ │ │ │ │ ├── SigninForm/ │ │ │ │ │ ├── components/ │ │ │ │ │ │ └── SubmitButton.tsx │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── SignupForm/ │ │ │ │ │ ├── components/ │ │ │ │ │ │ └── PiwikModal.tsx │ │ │ │ │ ├── form-validator.spec.ts │ │ │ │ │ ├── form-validator.ts │ │ │ │ │ ├── index.spec.tsx │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── UserInfo/ │ │ │ │ │ ├── UserInfo.spec.tsx │ │ │ │ │ ├── index.js │ │ │ │ │ ├── presenter.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── UserMenu/ │ │ │ │ │ ├── index.js │ │ │ │ │ ├── next-feature-button.scss │ │ │ │ │ ├── presenter.jsx │ │ │ │ │ ├── presenter.spec.jsx │ │ │ │ │ └── style.scss │ │ │ │ └── WithUser/ │ │ │ │ ├── index.js │ │ │ │ └── presenter.jsx │ │ │ ├── hoc/ │ │ │ │ └── withUser.jsx │ │ │ ├── hooks/ │ │ │ │ └── useUser.ts │ │ │ ├── index.ts │ │ │ ├── selectors/ │ │ │ │ └── userSelector.ts │ │ │ ├── services/ │ │ │ │ ├── deleteUser.js │ │ │ │ ├── isAuthenticated/ │ │ │ │ │ └── index.js │ │ │ │ ├── signup/ │ │ │ │ │ └── index.js │ │ │ │ └── usernameNormalizer/ │ │ │ │ └── index.js │ │ │ ├── store/ │ │ │ │ ├── index.ts │ │ │ │ ├── reducer.ts │ │ │ │ └── selectors.ts │ │ │ └── types.d.ts │ │ ├── userNotify/ │ │ │ ├── actions/ │ │ │ │ └── notify.js │ │ │ ├── hoc/ │ │ │ │ └── withNotification.jsx │ │ │ └── index.js │ │ └── view/ │ │ ├── actions/ │ │ │ └── requestMessages.js │ │ ├── components/ │ │ │ └── WithViewModel/ │ │ │ └── index.jsx │ │ ├── config.js │ │ ├── index.js │ │ ├── models/ │ │ │ └── View.js │ │ └── selectors/ │ │ └── viewSelector.js │ ├── scenes/ │ │ ├── About/ │ │ │ ├── components/ │ │ │ │ └── StylezedScreenshot/ │ │ │ │ └── index.tsx │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── ApplicationSettings/ │ │ │ ├── components/ │ │ │ │ ├── ContactSettings/ │ │ │ │ │ ├── index.js │ │ │ │ │ └── presenter.jsx │ │ │ │ ├── DesktopNotificationSettings/ │ │ │ │ │ ├── index.js │ │ │ │ │ ├── presenter.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── InterfaceSettings/ │ │ │ │ │ ├── index.js │ │ │ │ │ └── presenter.jsx │ │ │ │ ├── MessageSettings/ │ │ │ │ │ ├── index.js │ │ │ │ │ └── presenter.jsx │ │ │ │ └── NotificationSettings/ │ │ │ │ ├── index.js │ │ │ │ └── presenter.jsx │ │ │ ├── index.jsx │ │ │ ├── presenter.jsx │ │ │ └── style.scss │ │ ├── ContactAssociation/ │ │ │ ├── contact-association.scss │ │ │ ├── index.spec.tsx │ │ │ └── index.tsx │ │ ├── DevicesSettings/ │ │ │ ├── components/ │ │ │ │ ├── DeviceForm/ │ │ │ │ │ ├── index.spec.tsx │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── DeviceInformation/ │ │ │ │ │ ├── index.spec.tsx │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── DeviceSettings/ │ │ │ │ │ └── index.tsx │ │ │ │ ├── RevokeDevice/ │ │ │ │ │ ├── index.spec.tsx │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ └── VerifyDevice/ │ │ │ │ ├── index.spec.tsx │ │ │ │ ├── index.tsx │ │ │ │ └── style.scss │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── Discussion/ │ │ │ ├── components/ │ │ │ │ ├── AddParticipantsToContactBook/ │ │ │ │ │ ├── add-participants-dropdown.scss │ │ │ │ │ └── index.jsx │ │ │ │ ├── DownloadFileProgression/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── InstantMessage/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ ├── instant-message-aside.scss │ │ │ │ │ ├── instant-message-author.scss │ │ │ │ │ ├── instant-message-participants.scss │ │ │ │ │ └── style.scss │ │ │ │ ├── MailMessage/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ ├── mail-message-details.scss │ │ │ │ │ └── style.scss │ │ │ │ ├── Message.tsx │ │ │ │ ├── MessageAttachments/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── MessageList/ │ │ │ │ │ ├── index.js │ │ │ │ │ ├── presenter.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── MessagePi/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── MessageRecipients.tsx │ │ │ │ ├── ProtocolSwitch/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── QuickDraftForm/ │ │ │ │ │ ├── components/ │ │ │ │ │ │ └── ToggleAdvancedFormButton/ │ │ │ │ │ │ ├── index.jsx │ │ │ │ │ │ └── toggle-advanced-draft-button.scss │ │ │ │ │ ├── draft-message-quick.scss │ │ │ │ │ └── index.tsx │ │ │ │ ├── ReplyExcerpt/ │ │ │ │ │ ├── index.js │ │ │ │ │ ├── presenter.jsx │ │ │ │ │ └── style.scss │ │ │ │ └── TagList/ │ │ │ │ ├── index.jsx │ │ │ │ └── style.scss │ │ │ ├── discussion-action-bar.scss │ │ │ ├── index.js │ │ │ ├── presenter.jsx │ │ │ ├── style.scss │ │ │ └── types.d.ts │ │ ├── ForgotPassword/ │ │ │ ├── components/ │ │ │ │ └── ForgotPasswordForm/ │ │ │ │ ├── index.jsx │ │ │ │ ├── presenter.jsx │ │ │ │ └── style.scss │ │ │ ├── index.js │ │ │ ├── presenter.jsx │ │ │ └── presenter.spec.jsx │ │ ├── NewDeviceInfo/ │ │ │ ├── index.jsx │ │ │ └── style.scss │ │ ├── NewDraft/ │ │ │ ├── components/ │ │ │ │ ├── DraftDiscussion/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── useDraftDiscussion.tsx │ │ │ │ └── DraftMessage/ │ │ │ │ ├── components/ │ │ │ │ │ ├── IdentitySelector.tsx │ │ │ │ │ └── Recipients.tsx │ │ │ │ ├── draft-message-advanced.scss │ │ │ │ ├── draft-message-placeholder.scss │ │ │ │ └── index.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── RemoteIdentitySettings/ │ │ │ ├── components/ │ │ │ │ ├── AuthButton/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── LastConnection/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── NewIdentity/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ ├── new-identity.scss │ │ │ │ │ ├── provider-email-button.scss │ │ │ │ │ └── provider-list.scss │ │ │ │ ├── ProviderButton/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── ProviderButtonContainer/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── RemoteIdentity/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── RemoteIdentityDetails/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── RemoteIdentityEmail/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ ├── index.spec.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── RemoteIdentityOauth/ │ │ │ │ │ └── index.jsx │ │ │ │ └── Status/ │ │ │ │ └── index.jsx │ │ │ ├── index.js │ │ │ ├── presenter.jsx │ │ │ └── style.scss │ │ ├── ResetPassword/ │ │ │ ├── components/ │ │ │ │ └── ResetPasswordForm/ │ │ │ │ ├── index.jsx │ │ │ │ ├── presenter.jsx │ │ │ │ └── style.scss │ │ │ ├── index.js │ │ │ ├── presenter.jsx │ │ │ └── presenter.spec.jsx │ │ ├── SearchResults/ │ │ │ ├── components/ │ │ │ │ ├── ContactResultItem/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── Highlights/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ ├── index.spec.jsx │ │ │ │ │ └── style.scss │ │ │ │ └── MessageResultItem/ │ │ │ │ ├── index.tsx │ │ │ │ └── style.scss │ │ │ ├── index.js │ │ │ ├── presenter.jsx │ │ │ └── style.scss │ │ ├── SettingsSignatures/ │ │ │ ├── components/ │ │ │ │ └── SignatureForm/ │ │ │ │ ├── index.jsx │ │ │ │ └── presenter.jsx │ │ │ ├── index.jsx │ │ │ ├── presenter.jsx │ │ │ └── style.scss │ │ ├── Signin.tsx │ │ ├── Signup.tsx │ │ ├── TagsSettings/ │ │ │ ├── components/ │ │ │ │ ├── TagInput/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ └── TagSearch/ │ │ │ │ ├── index.tsx │ │ │ │ └── style.scss │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── Timeline/ │ │ │ ├── components/ │ │ │ │ ├── DiscussionItem/ │ │ │ │ │ ├── discussion-item-content.scss │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ └── DiscussionSelector/ │ │ │ │ ├── index.jsx │ │ │ │ └── style.scss │ │ │ ├── index.js │ │ │ ├── presenter.jsx │ │ │ ├── style.scss │ │ │ └── timeline-action-bar.scss │ │ ├── UserPrivacy/ │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── UserProfile/ │ │ │ ├── components/ │ │ │ │ ├── ProfileForm/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ └── ProfileInfo/ │ │ │ │ ├── index.tsx │ │ │ │ └── style.scss │ │ │ ├── index.spec.tsx │ │ │ ├── index.tsx │ │ │ └── style.scss │ │ ├── UserSecurity/ │ │ │ ├── components/ │ │ │ │ ├── LoginDetails/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── OpenPGPForm.scss │ │ │ │ ├── OpenPGPGenerateForm.tsx │ │ │ │ ├── OpenPGPImportForm.tsx │ │ │ │ ├── OpenPGPKey/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── OpenPGPPrivateKeys/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── PasswordDetails/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ ├── PasswordForm/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── style.scss │ │ │ │ └── TFAForm/ │ │ │ │ └── index.jsx │ │ │ ├── index.jsx │ │ │ ├── presenter.jsx │ │ │ └── style.scss │ │ ├── ValidateDevice/ │ │ │ ├── index.js │ │ │ ├── presenter.jsx │ │ │ └── style.scss │ │ ├── View/ │ │ │ ├── components/ │ │ │ │ ├── MessageItem/ │ │ │ │ │ ├── index.js │ │ │ │ │ ├── presenter.jsx │ │ │ │ │ └── style.scss │ │ │ │ └── MessageSelector/ │ │ │ │ ├── index.jsx │ │ │ │ └── style.scss │ │ │ ├── index.jsx │ │ │ ├── style.scss │ │ │ └── withCurrentView.jsx │ │ ├── contact/ │ │ │ ├── Contact.tsx │ │ │ ├── ContactBook/ │ │ │ │ ├── components/ │ │ │ │ │ ├── ImportContact/ │ │ │ │ │ │ ├── index.js │ │ │ │ │ │ └── presenter.jsx │ │ │ │ │ ├── ImportContactButton/ │ │ │ │ │ │ └── index.tsx │ │ │ │ │ ├── ImportContactForm/ │ │ │ │ │ │ ├── index.jsx │ │ │ │ │ │ ├── presenter.jsx │ │ │ │ │ │ └── style.scss │ │ │ │ │ └── TagList/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── contact-book-menu.scss │ │ │ │ ├── index.spec.tsx │ │ │ │ ├── index.tsx │ │ │ │ ├── query.spec.tsx │ │ │ │ ├── query.ts │ │ │ │ └── style.scss │ │ │ ├── ContactForm.tsx │ │ │ ├── EditContact.spec.tsx │ │ │ ├── EditContact.tsx │ │ │ ├── NewContact.spec.tsx │ │ │ ├── NewContact.tsx │ │ │ ├── components/ │ │ │ │ ├── AddFormFieldForm/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── AddressDetails/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── AddressForm/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── BirthdayDetails/ │ │ │ │ │ └── index.tsx │ │ │ │ ├── BirthdayForm/ │ │ │ │ │ ├── index.ts │ │ │ │ │ └── presenter.tsx │ │ │ │ ├── ContactPageWrapper.tsx │ │ │ │ ├── ContactProfileForm/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── ContactTitleField/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── EmailDetails/ │ │ │ │ │ └── index.tsx │ │ │ │ ├── EmailForm/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── FormCollection/ │ │ │ │ │ └── index.tsx │ │ │ │ ├── IdentityDetails/ │ │ │ │ │ └── index.tsx │ │ │ │ ├── IdentityForm/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── ImDetails/ │ │ │ │ │ └── index.tsx │ │ │ │ ├── ImForm/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── OrgaDetails/ │ │ │ │ │ └── index.tsx │ │ │ │ ├── OrgaForm/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── PhoneDetails/ │ │ │ │ │ └── index.tsx │ │ │ │ ├── PhoneForm/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── PublicKeyForm/ │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── presenter.tsx │ │ │ │ │ └── style.scss │ │ │ │ ├── PublicKeyList/ │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── style.scss │ │ │ │ └── ReduxedInputFileGroup/ │ │ │ │ ├── index.tsx │ │ │ │ └── style.scss │ │ │ ├── contact-action-bar.scss │ │ │ ├── contact-main-title.scss │ │ │ ├── services/ │ │ │ │ ├── contactValidation.jsx │ │ │ │ ├── contactValidation.spec.jsx │ │ │ │ ├── handleContactSaveErrors.spec.ts │ │ │ │ └── handleContactSaveErrors.ts │ │ │ └── style.scss │ │ └── error/ │ │ ├── PageError.tsx │ │ ├── PageNotFound.tsx │ │ ├── components/ │ │ │ └── CaliopenAscii.tsx │ │ └── style.scss │ ├── services/ │ │ ├── api-client/ │ │ │ ├── index.ts │ │ │ └── types.d.ts │ │ ├── api-location/ │ │ │ └── index.js │ │ ├── api-patch/ │ │ │ ├── index.js │ │ │ └── index.spec.js │ │ ├── asciify.js │ │ ├── asciify.spec.js │ │ ├── browser-notification/ │ │ │ └── index.js │ │ ├── capitalize.js │ │ ├── config.ts │ │ ├── encode-utils/ │ │ │ ├── index.js │ │ │ └── utf8ArrayToString.js │ │ ├── encryption/ │ │ │ ├── index.js │ │ │ └── index.spec.js │ │ ├── event-manager/ │ │ │ └── index.js │ │ ├── filter-contacts/ │ │ │ ├── index.js │ │ │ └── index.spec.js │ │ ├── importance-level/ │ │ │ └── index.js │ │ ├── localStorage/ │ │ │ └── index.js │ │ ├── message/ │ │ │ ├── index.js │ │ │ └── index.spec.js │ │ ├── mime/ │ │ │ └── index.js │ │ ├── openpgp-keychain-repository/ │ │ │ └── index.js │ │ ├── openpgp-manager/ │ │ │ ├── api.ts │ │ │ ├── errors.ts │ │ │ ├── index.spec.ts │ │ │ └── index.ts │ │ ├── protocols-config/ │ │ │ └── index.ts │ │ ├── renderReduxField/ │ │ │ └── index.jsx │ │ └── username-utils/ │ │ ├── username-availability.ts │ │ ├── username-validity.spec.ts │ │ └── username-validity.ts │ ├── store/ │ │ ├── actions/ │ │ │ ├── message.js │ │ │ └── timeline.js │ │ ├── configure-store.ts │ │ ├── middlewares/ │ │ │ ├── axios-middleware.ts │ │ │ ├── crash-reporter-middleware.js │ │ │ ├── decryption-middleware.js │ │ │ ├── discussions-middleware.js │ │ │ ├── encryption-middleware.js │ │ │ ├── freeze.js │ │ │ ├── importance-level-middleware.js │ │ │ ├── messages-middleware.js │ │ │ └── search-middleware.js │ │ ├── modules/ │ │ │ ├── device.ts │ │ │ ├── discussion.ts │ │ │ ├── draft-message.spec.ts │ │ │ ├── draft-message.ts │ │ │ ├── encryption.ts │ │ │ ├── i18n.js │ │ │ ├── importance-level.js │ │ │ ├── local-identity.js │ │ │ ├── message.js │ │ │ ├── message.spec.js │ │ │ ├── notification.js │ │ │ ├── participant-suggestions.ts │ │ │ ├── provider.js │ │ │ ├── public-key.js │ │ │ ├── remote-identity.js │ │ │ ├── search.js │ │ │ ├── settings.js │ │ │ └── view.js │ │ ├── reducer.ts │ │ └── selectors/ │ │ ├── device.js │ │ ├── getModuleStateSelector.js │ │ ├── message.js │ │ ├── provider.js │ │ ├── settings.js │ │ ├── tab.js │ │ └── timeline.js │ ├── styles/ │ │ ├── base/ │ │ │ ├── fonts.scss │ │ │ └── reset.scss │ │ ├── common.scss │ │ ├── config/ │ │ │ ├── config.scss │ │ │ └── theme/ │ │ │ ├── dark.scss │ │ │ └── light.scss │ │ ├── object/ │ │ │ ├── o-avatar.scss │ │ │ ├── o-callout.scss │ │ │ ├── o-clickable.scss │ │ │ ├── o-date-picker.scss │ │ │ ├── o-form-element.scss │ │ │ ├── o-hidden-element.scss │ │ │ ├── o-navigation.scss │ │ │ ├── o-pi-border.scss │ │ │ ├── o-reset-html-message.scss │ │ │ └── o-triangle.scss │ │ ├── util/ │ │ │ ├── breakpoint.scss │ │ │ ├── flex-grid.scss │ │ │ └── u-gradient.scss │ │ └── vendor/ │ │ ├── bootstrap_font-awesome.scss │ │ ├── bootstrap_foundation-sites.scss │ │ ├── bootstrap_react-redux-notify.scss │ │ └── foundation-sites/ │ │ ├── _config.scss │ │ └── _flex-grid.scss │ └── types.d.ts ├── template/ │ ├── index.ejs │ └── index.html ├── test/ │ ├── fixtures/ │ │ ├── contacts/ │ │ │ ├── data.json │ │ │ └── index.ts │ │ ├── device/ │ │ │ └── index.ts │ │ ├── tags/ │ │ │ └── data.json │ │ └── user/ │ │ ├── data.json │ │ └── index.ts │ ├── functional/ │ │ ├── features/ │ │ │ ├── 00_home-spec.js │ │ │ ├── 10_discussion-spec.js │ │ │ ├── 20-message-list-spec.js │ │ │ ├── 21-reply-to-messag-spec.js │ │ │ ├── 22-compose-spec.js │ │ │ ├── 23-delete-message-spec.js │ │ │ ├── 24-scroll-spec.js │ │ │ ├── 25-assoc-participant-contact-spec.js │ │ │ ├── 30-settings-spec.js │ │ │ ├── 35-remote-identitiy-settings-spec.js │ │ │ ├── 40-new-contact-spec.js │ │ │ ├── 50-search-spec.js │ │ │ ├── 60-tag-spec.js │ │ │ ├── 70-device-spec.js │ │ │ ├── 80-account-spec.js │ │ │ └── 90_notification-spec.js │ │ ├── protractor.conf.js │ │ └── utils/ │ │ ├── navigation.js │ │ ├── timeline.js │ │ ├── user-util.js │ │ └── window.js │ ├── msw-handlers/ │ │ ├── contacts.ts │ │ ├── settings.ts │ │ ├── tags.ts │ │ └── user.ts │ ├── providers.tsx │ ├── server.ts │ └── unit/ │ ├── lingui-react.jsx │ └── setup.ts ├── tsconfig.json └── webpack/ ├── config.js ├── webpack.common.js ├── webpack.config.browser.js └── webpack.config.server.js ================================================ FILE CONTENTS ================================================ ================================================ FILE: .drone.yml ================================================ workspace: base: /srv path: caliopen clone: git: image: plugins/git depth: 10 recursive: false pipeline: ################################## ## BASE IMAGES ## ################################## build-caliopen-go: group: build0 image: public-registry.caliopen.org/caliopen_drone_docker privileged: true secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY] environment: - DEPS=vendor - BASE_DIR=src/backend - PLUGIN_DOCKERFILE=src/backend/Dockerfile.caliopen-go - PLUGIN_CONTEXT=/srv/caliopen/src/backend - PLUGIN_REPO=registry.caliopen.org/caliopen_go when: branch: [ develop, master ] event: [ push ] commands: - . devtools/drone/files_changed.sh - . devtools/drone/build_images.sh ####################################################### ## TEST ON MASTER OR DEVELOP PR ## ####################################################### test-go: group: test pull: true image: public-registry.caliopen.org/caliopen_go environment: - BASE_DIR=/go/src/github.com/CaliOpen/Caliopen/src/backend when: branch: [ develop ] event: [ pull_request ] commands: - cp -r /srv/caliopen/src/backend/* $${BASE_DIR} - cd $${BASE_DIR} && govendor sync -v - go test ./... test-py: pull: true group: test image: public-registry.caliopen.org/caliopen_python environment: - CALIOPEN_BASEDIR=/srv/caliopen - BASE_DIR=src/backend when: branch: [ develop, master ] event: [ pull_request ] commands: - . devtools/drone/files_changed.sh - . devtools/drone/test_py.sh test-frontend: group: test image: node:16 when: branch: [ develop, master ] event: [ pull_request ] environment: - BASE_DIR=src/frontend commands: - . devtools/drone/files_changed.sh - . devtools/drone/test_front.sh ######################################################### ## BUILD AND PUBLISH ON DEVELOP PUSH ## ######################################################### # Python develop images build-py-cli-develop: group: build1 image: public-registry.caliopen.org/caliopen_drone_docker privileged: true secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY] environment: - PLUGIN_DOCKERFILE=src/backend/Dockerfile.cli - PLUGIN_CONTEXT=/srv/caliopen/src/backend - PLUGIN_REPO=registry.caliopen.org/caliopen_cli - BASE_DIR=src/backend - LANG=python - PROG=tools/py.CLI when: branch: [ develop ] event: [ push ] commands: - export PLUGIN_TAGS=develop - . devtools/drone/get_py_dependencies.sh # Get the list of dependencies - . devtools/drone/files_changed.sh # Check if any file has been modified - . devtools/drone/build_images.sh # Build docker image build-apiv1-develop: group: build1 image: public-registry.caliopen.org/caliopen_drone_docker privileged: true secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY] environment: - PLUGIN_DOCKERFILE=src/backend/Dockerfile.py-api - PLUGIN_CONTEXT=/srv/caliopen/src/backend - PLUGIN_REPO=registry.caliopen.org/caliopen_apiv1 - BASE_DIR=src/backend - LANG=python - PROG=interfaces/REST/py.server when: branch: [ develop ] event: [ push ] commands: - export PLUGIN_TAGS=develop - . devtools/drone/get_py_dependencies.sh # Get the list of dependencies - . devtools/drone/files_changed.sh # Check if any file has been modified - . devtools/drone/build_images.sh # Build docker image build-mq-worker-develop: group: build1 image: public-registry.caliopen.org/caliopen_drone_docker privileged: true secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY] environment: - PLUGIN_DOCKERFILE=src/backend/Dockerfile.mq-worker - PLUGIN_CONTEXT=/srv/caliopen/src/backend - PLUGIN_REPO=registry.caliopen.org/caliopen_mqworker - BASE_DIR=src/backend - LANG=python - PROG=interfaces/NATS/py.client when: branch: [ develop ] event: [ push ] commands: - export PLUGIN_TAGS=develop - . devtools/drone/get_py_dependencies.sh # Get the list of dependencies - . devtools/drone/files_changed.sh # Check if any file has been modified - . devtools/drone/build_images.sh # Build docker image ## GO develop images build-apiv2-develop: group: build2 image: public-registry.caliopen.org/caliopen_drone_docker privileged: true secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY] environment: - PLUGIN_DOCKERFILE=src/backend/Dockerfile.go-api - PLUGIN_CONTEXT=/srv/caliopen/src/backend - PLUGIN_REPO=registry.caliopen.org/caliopen_apiv2 - PROG=interfaces/REST/go.server/cmd/caliopen_rest - BASE_DIR=src/backend - LANG=go when: branch: [ develop ] event: [ push ] commands: - export PLUGIN_TAGS=develop - . devtools/drone/get_go_dependencies.sh - . devtools/drone/files_changed.sh - . devtools/drone/build_images.sh build-lmtpd-develop: group: build2 image: public-registry.caliopen.org/caliopen_drone_docker privileged: true secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY] environment: - PLUGIN_DOCKERFILE=src/backend/Dockerfile.go-lmtp - PLUGIN_CONTEXT=/srv/caliopen/src/backend - PLUGIN_REPO=registry.caliopen.org/caliopen_lmtpd - PROG=protocols/go.smtp/cmd/caliopen_lmtpd - BASE_DIR=src/backend - LANG=go when: branch: [ develop ] event: [ push ] commands: - export PLUGIN_TAGS=develop - . devtools/drone/get_go_dependencies.sh - . devtools/drone/files_changed.sh - . devtools/drone/build_images.sh build-idpoller-develop: group: build2 image: public-registry.caliopen.org/caliopen_drone_docker privileged: true secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY] environment: - PLUGIN_DOCKERFILE=src/backend/Dockerfile.identity-poller - PLUGIN_CONTEXT=/srv/caliopen/src/backend - PLUGIN_REPO=registry.caliopen.org/caliopen_identitypoller - PROG=workers/go.remoteIDs/cmd/idpoller - BASE_DIR=src/backend - LANG=go when: branch: [ develop ] event: [ push ] commands: - export PLUGIN_TAGS=develop - . devtools/drone/get_go_dependencies.sh - . devtools/drone/files_changed.sh - . devtools/drone/build_images.sh build-imapworker-develop: group: build2 image: public-registry.caliopen.org/caliopen_drone_docker privileged: true secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY] environment: - PLUGIN_DOCKERFILE=src/backend/Dockerfile.imap-worker - PLUGIN_CONTEXT=/srv/caliopen/src/backend - PLUGIN_REPO=registry.caliopen.org/caliopen_imapworker - PROG=protocols/go.imap/cmd/imapworker - BASE_DIR=src/backend - LANG=go when: branch: [ develop ] event: [ push ] commands: - export PLUGIN_TAGS=develop - . devtools/drone/get_go_dependencies.sh - . devtools/drone/files_changed.sh - . devtools/drone/build_images.sh build-twitterworker-develop: group: build2 image: public-registry.caliopen.org/caliopen_drone_docker privileged: true secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY] environment: - PLUGIN_DOCKERFILE=src/backend/Dockerfile.twitter-worker - PLUGIN_CONTEXT=/srv/caliopen/src/backend - PLUGIN_REPO=registry.caliopen.org/caliopen_twitterworker - PROG=protocols/go.twitter/cmd/twitterworker - BASE_DIR=src/backend - LANG=go when: branch: [ develop ] event: [ push ] commands: - export PLUGIN_TAGS=develop - . devtools/drone/get_go_dependencies.sh - . devtools/drone/files_changed.sh - . devtools/drone/build_images.sh build-frontend-develop: group: build3 image: public-registry.caliopen.org/caliopen_drone_docker privileged: true secrets: [ DOCKER_USERNAME , DOCKER_PASSWORD, DOCKER_REGISTRY ] environment: - PLUGIN_DOCKERFILE=src/frontend/web_application/Dockerfile - PLUGIN_CONTEXT=/srv/caliopen/src/frontend/web_application - PLUGIN_REPO=registry.caliopen.org/caliopen_frontend - BASE_DIR=src/frontend/web_application - LANG=js when: branch: [ develop ] event: [ push ] commands: - export PLUGIN_TAGS=develop - . devtools/drone/files_changed.sh - . devtools/drone/build_images.sh ######################################################### ## BUILD AND PUBLISH ON TAG: RELEASE- ## ######################################################### build-py-cli-release: group: release1 image: plugins/docker dockerfile: src/backend/Dockerfile.cli context: /srv/caliopen/src/backend repo: registry.caliopen.org/caliopen_cli secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY ] when: ref: [ "refs/tags/release-*" ] event: [ tag ] tags: - latest - ${DRONE_TAG##release-} build-apiv1-release: group: release1 image: plugins/docker dockerfile: src/backend/Dockerfile.py-api context: /srv/caliopen/src/backend repo: registry.caliopen.org/caliopen_apiv1 secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY ] when: ref: [ "refs/tags/release-*" ] event: [ tag ] tags: - latest - ${DRONE_TAG##release-} build-mq-worker-release: group: release1 image: plugins/docker dockerfile: src/backend/Dockerfile.mq-worker context: /srv/caliopen/src/backend repo: registry.caliopen.org/caliopen_mqworker secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY ] when: ref: [ "refs/tags/release-*" ] event: [ tag ] tags: - latest - ${DRONE_TAG##release-} build-apiv2-release: group: release2 image: plugins/docker dockerfile: src/backend/Dockerfile.go-api context: /srv/caliopen/src/backend repo: registry.caliopen.org/caliopen_apiv2 secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY ] when: ref: [ "refs/tags/release-*" ] event: [ tag ] tags: - latest - ${DRONE_TAG##release-} build-lmtpd-release: group: release2 image: plugins/docker dockerfile: src/backend/Dockerfile.go-lmtp context: /srv/caliopen/src/backend repo: registry.caliopen.org/caliopen_lmtpd secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY ] when: ref: [ "refs/tags/release-*" ] event: [ tag ] tags: - latest - ${DRONE_TAG##release-} build-idpoller-release: group: release2 image: plugins/docker dockerfile: src/backend/Dockerfile.identity-poller context: /srv/caliopen/src/backend repo: registry.caliopen.org/caliopen_identitypoller secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY ] when: ref: [ "refs/tags/release-*" ] event: [ tag ] tags: - latest - ${DRONE_TAG##release-} build-imapworker-release: group: release2 image: plugins/docker dockerfile: src/backend/Dockerfile.imap-worker context: /srv/caliopen/src/backend repo: registry.caliopen.org/caliopen_imapworker secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY ] when: ref: [ "refs/tags/release-*" ] event: [ tag ] tags: - latest - ${DRONE_TAG##release-} build-twitterworker-release: group: release2 image: plugins/docker dockerfile: src/backend/Dockerfile.twitter-worker context: /srv/caliopen/src/backend repo: registry.caliopen.org/caliopen_twitterworker secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY ] when: ref: [ "refs/tags/release-*" ] event: [ tag ] tags: - latest - ${DRONE_TAG##release-} build-frontend-release: group: release3 image: plugins/docker dockerfile: src/frontend/web_application/Dockerfile context: /srv/caliopen/src/frontend/web_application repo: registry.caliopen.org/caliopen_frontend secrets: [ DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REGISTRY ] when: ref: [ "refs/tags/release-*" ] event: [ tag ] tags: - latest - ${DRONE_TAG##release-} ######################################################### ## SERVICES ## ######################################################### ================================================ FILE: .git-crypt/.gitattributes ================================================ # Do not edit this file. To specify the files to encrypt, create your own # .gitattributes file in the directory where your files are. * !filter !diff *.gpg binary ================================================ FILE: .gitattributes ================================================ devtools/kubernetes/secrets/*.yaml filter=git-crypt diff=git-crypt ================================================ FILE: .gitignore ================================================ # Created by .ignore support plugin (hsz.mobi) ### JetBrains template # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff: .idea/workspace.xml .idea/tasks.xml .idea/dictionaries .idea/vcs.xml .idea/jsLibraryMappings.xml # Sensitive or high-churn files: .idea/dataSources.ids .idea/dataSources.xml .idea/dataSources.local.xml .idea/sqlDataSources.xml .idea/dynamic.xml .idea/uiDesigner.xml # Gradle: .idea/gradle.xml .idea/libraries # Mongo Explorer plugin: .idea/mongoSettings.xml ## File-based project format: *.iws ## Plugin-specific files: # IntelliJ /out/ # mpeltonen/sbt-idea plugin .idea_modules/ # JIRA plugin atlassian-ide-plugin.xml # Crashlytics plugin (for Android Studio and IntelliJ) com_crashlytics_export_strings.xml crashlytics.properties crashlytics-build.properties fabric.properties ### VirtualEnv template # Virtualenv # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ .Python [Ii]nclude [Ll]ib64 [Ll]ocal [Ss]cripts pyvenv.cfg .venv pip-selfcheck.json ### Python template # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *,cover .hypothesis/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # IPython Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # dotenv .env # virtualenv venv/ ENV/ # Spyder project settings .spyderproject # Rope project settings .ropeproject /.idea/ *.iml # vagrant directory .vagrant # Docker data .data # Golang src/backend/**/vendor/**/ # Node node_modules # private fixtures /devtools/email-with-parts-and-attachments # registry configuration /devtools/registry.conf ================================================ FILE: .travis.yml ================================================ sudo: required services: - docker env: COMPOSE_VERSION: 1.9.0 language: node_js node_js: - '8' cache: - yarn git: depth: 10 before_install: - curl -L https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - curl -o- -L https://yarnpkg.com/install.sh | bash - export PATH="$HOME/.yarn/bin:$PATH" before_script: - git fetch origin $TRAVIS_BRANCH:$TRAVIS_BRANCH script: - devtools/run-tests.sh notifications: irc: channels: - "chat.freenode.net#caliopdev" ================================================ FILE: CHANGELOG.md ================================================ # Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] ## [0.27.1] 2023-05-09 ### Fixed - JS error when saving settings. ## [0.27.0] 2023-05-05 ### Changed - CLI commands `resync_shard` and `resync_index` does not stop when an error occured. ## [0.26.1] 2023-05-03 ### Fixed - Empty discussion list. ## [0.26.0] 2023-04-04 ### Changed - Replace PI message illustration by a PI score (A to E). ## [0.25.3] 2023-02-28 ### Fixed - Improve device management frontend codebase ## [0.25.2] 2023-01-25 ### Fixed - Handle username availability errors on the client. - Remove unexpected errors on login page on first display. ## [0.25.1] 2023-01-12 ### Fixed - Update client's dependencies. - Load locale at run time on the client. - Small improvements in tag managements & Contact book. ## [0.25.0] 2022-03-03 ### Change - Improve contact edit form. Better error handling. ## [0.24.6] 2021-10-23 ### Fixed - Unable to to delete account. ## [0.24.5] 2021-08-20 ### Fixed - Crash after signup, use a subquery instead. ### Changed - Redirect to external account configuration after signup. ## [0.24.4] 2021-07-25 ### Fixed - Advanced draft suggestions correctly displayed. - Color of signin & signup spinner button set to bright. ## [0.24.3] 2021-06-14 ### Fixed - Unable to quick reply to a mastodon or tweeter dm ## [0.24.2] 2021-06-11 ### Fixed - client BSOD: Make sure user is loaded before accessing to the contact. ## [0.24.1] 2021-06-09 ### Fixed - tag management - login form - Mastodon DM handling ## [0.24.0] 2021-03-02 ### Changed - re-enable Take a Tour. - update internal frontend dependencies. - Edit advanced draft in its specific page. - update dependencies ### Fixed - Openpgp form when only one identity. ## [0.23.1] 2019-09-17 ### Fixed - No feedbacks were displayed when errors happend on signin form. - No spinner on activity on signin form. ## [0.23] 2019-07-19 ### Added - Mastodon protocol - Tag email with imap flags when fetching external account ### Changed - Better Timeline display for large and small screens. - Batch notifications ### Fixed - bugs in imap worker - bugs in some responsiveness UI components - username validation ## [0.22] 2019-06-26 ## Added - The new about landpage. ### Changed - Upload PGP keys using files rather than textboxes. - Notifications : output bach operations (external identities fetches for ex.) into one notification ### Fixed - Double redirections algorithm client and server when not authenticated - Prevent target blank anchor's xss. - Reload devices when changing page or when invalidated. - Page rendering after signout with unexpected content cached. - username validation to conform more tightly to specifications. ## [0.21] 2019-06-06 ### Added - Always link participant to contact in a draft when contact exists (ensure message can be encrypted) - Validate body cannot be empty for a Twitter DM. - Upload user's public key when adding a private key. - Discussion has unread message button. - Dynamic contacts references embedded in messages ### Changed - FTS and participants lookup improvments - On quick reply, «Enter» will not send draft anymore, it must be CTRL+Enter. - Quick reply is now multilines. - Use message's excerpt in search results instead of garbled highlights - Move vcard file import route on apiv2 to use contact uniqueness and lookups principles - Re-enable import contacts via vcard file. - New messages notification is not displayed anymore, it is now automatically loaded. - Send button icon in advanced form. - Disable «Take a tour» which is not working properly - Disable the send button on quick draft when empty (thanks Sebbaz). - Look and feel of GPG buttons which look like disabled. ### Fixed - Default locale saved on account creation. - Responsiveness for the «new device page». - BSOD while including regexp special chars in search query. - Email icon was always on Timeline. - Contact starting w/ a letter with a diacritic is not displayed on ContactBook. - Contacts list update after contact deletion - Twitter workers errors handling - Imap workers errors handling - Messages' excerpts showing html tags - BSOD when changing identity in draft when no recipients (thanks peha) - Many encoding issues during mail delivery - Participants algorithm edge case - Discussion last message sort better - Index user contact without alias using a workaround to bad core/mixin classes design - Addresses emails parsing failed sometimes with strange values - Process better invalid or missing data and encoding problems in incoming message - Signout wasn't effective ## [0.20.0] 2019-05-15 ### Added - Validation of twitter username in contact edition - Address (and protocol) selection in a 1-to-1 discussion - Activate links in plain text messages. - Handle client crashes and provide a link to report an issue on https://feedback.caliopen.org ### Changed - Disable draft form in case there is no selected identity - Facebook username is no more available in contact edition - In a draft, switch identity will change the protocol of all recipients as well - Disable contact import - Do not display private key details, allow "download" instead. - Providers buttons are available according to backend configuration (api: `/api/v2/providers`) ### Fixed - Simple detection of PGP inline message - Apiv2 create and delete contacts does not use `user.shard_id` - Better logging for apiv2 and mq-worker - Do not fail if ContactLookup raise a NotFound - Twitter nick not displayed on contact book - Select Twitter identity according to parent message on a new draft - Show at least 1 participant per discussion on timeline - Send a quick draft by pressing «Enter» and display a spinner ## [0.18.2] 2019-04-26 ### Fixed - signup does not authenticate and crash - hardcoded references to "alpha.caliopen.org" in client ## [0.18.1] 2019-04-26 ### Changed - Better display of message's participants - Fix attachment visibility - Raise an explicit error on duplicate message for better processing - Validate better email address when cleaning it - Better oauth token validation ### Fixed - Attachments visibility, it displays a warning if the message has been encrypted ## [0.18.0] 2019-04-23 ### Fixed - render form after draft deletion - Reply encrypted messages - Hide "load" more button when all messages are displayed - Display decryption error message - Glitches on encrypted mails - Display facebook names on contact book ### Added - Send and receive text/plain MIME messages - Show full date on Hover - Suggest Twitter handles in draft form ## [0.17.0] 2019-03-21 ### Added - End to end PGP encryption/decryption. - When user adds an external account, external identity is added to user's contact card - Test for imap worker, twitter worker and identities worker - Actions for instant messages (delete, reply …) and tag list where missing - PWA: add to home screen - PWA: basic offline capabilities - Search when click on tags (basic search) - Verify device by mail ### Fixed - Fix chronological order of messages in discussion scene - Better responsiveness on small screens for timeline & discussion & logo - Better responsiveness on small screens for dropdowns - inversion en/fr for some translations - BSOD on draft view in case there is no author (for example after remote identity deletion) - Last messages not visible in case the discussion has been openned - Sync contact associated to the user when editing in the contact book - Accept only Private Key in user account ### Changed - Remove device's locations field (IP definition) that wasn't saved and has no effects for now ## [0.16.0] 2019-02-25 ### Added - Handle lost authentication, redirect signin - A simple view with draft messages - Display the discussion related to the selected participants of a draft - Compute an experimental different PI structure for message entity - A caliopen_data python package for caliopen data manipulation - A machine learning model for message automatic tagging - Create an RFC 3156 compatible mime structure for PGP encrypted sent email - Add an API route to find if a discussion exist for a list of participants - Ignore already imported message ### Changed - Move link external accounts to user menu - Smaller font for desktop - Use white color for plain text buttons - Refactor hover and active colors for buttons - Display a progress bar when downloading an attachment - New calcul for Privacy Index - Take A Tour has been moved in the new menu «Help & Info» - Display related emails for a PGP public key - Compute related discussion only when the message is sent - Better discussion match if any participant is a known contact - Reworked deeply job dispatching logic for protocol workers ### Fixed - Bad redirection when canceling contact creation - Add spaces between buttons in contact edit & contact association pages - Unable to download an attachment due to missing request's headers ## [0.15.2] 2019-01-23 ### Fixed - Many fixes on remote identities workers (imap, twitter) tested on production ## [0.15.1] 2019-01-22 ### Fixed - BSOD on discussion when contacts not yet loaded ## [0.15.0] 2019-01-15 ### Added - When an user authenticate we issue a device.login event - Add a contact from a discussion - Privacy Policy page available at https://alpha.caliopen.org/privacy-policy.html ### Fixed - typos on English catalog (thanks octplane) - Existing device are found better, lead to less untrusted device for user - Click on contact's title "input" submits the form - Empty contact book even when user has a contact (always actually) - Route /discussion/{discussion_id} to retrieve one discussion - `unread_count` in discussions list ### Changed - Rework how privacy features are declared and managed - Change how a discussion hash is build, take contact_id as better key ## [0.14.0] 2018-12-19 ### Added - special group user contact in contact-book - Toggle show spam - Placeholder when loading Timeline - Messages de-duplication when importing or re-importing from external accounts ### Fixed - Timeline responsiveness (dates & action bar) - Display last protocol used for a discussion in the timeline - Automatic set read when displaying a message - participants added from suggestions always define `email` protocol - More permissive protocol validation in draft according to identities ### Changed - Timeline colors ## [0.13.2] 2018-12-11 ### Added - Switch identity, send twitter DM, validation… - Support new dm twitter notification ### Fixed - Crash when replying a message. - Help button has no effects - Unable to change password - Better layout for twitter DM - Avatar size in tabs for twitter DM - Splash screen initialization after reconnect ## [0.13.1] 2018-12-05 ### Fixed - BSOD when selecting contact - All messages detected as Twitter DM - Missing user_identity when sending the reset password notification - Legacy protocol values are considered as valid ## [0.13.0] 2018-11-30 ### Added - API to manage cryptographic public keys related to a contact - API to list known remote identity providers - Create remote identity for gmail and twitter using Oauth mechanisms - A new worker to fetch twitter direct messages - Support touch scroll on navigation tabs - Manage public keys of a contact - App loader splash screen ### Changed - Brand new UI - Message Timeline replaced by Discussion Timeline - IM address for a contact is more permisive ### Removed - Sidescreen on small screen - In discussion messages grouped by date ### Fixed - Translation of new device screen in English ## [0.12.3] 2018-10-26 ### Fixed - Manage better how to declare an user index on signup - Index contact and message with correct user_id ### Changed - Build go images with a vendor sync and with CA certificates for TLS connection ## [0.12.2] 2018-10-15 ### Fixed - /suggest apiv2 route use user.shard_id index not user.user_id - message python object use user not user_id ## [0.12.1] 2018-10-04 ### Fixed - Share an index for many users, scalibility of elasticsearch does not work using old scheme ### Added - Add an email or a social identity to a contact trigger PGP key discovery process - A connection is possible per device ### Changed - prevent invalidation of whole discussion during scroll - LocalIdentities and RemoteIdentities have been merged into a new UserIdentity object - group discussion by list or all participants - API output more informations on discussions ### Fixed - message sort on "Load More" in discussion view - revert mis-deleted signup and recovery mail links ## [0.11.2] 2018-07-16 ### Fixed - load more doesn't load correctly when filter has been changed - prevent signin until JS is fully loaded, previously a json shows up with informations about a fake device. - unlock correctly syncing state after a fetch failure - safer parsing of email with ',' or '\r' character ### Changed - remove unused safe/public/unsafe login buttons ## [0.11.1] 2018-07-12 ### Added - explain how to authorize retrieve of an imap gmail account ### Fixed - do not try to parse a null date_sort - ensure to not retrieve credentials when patching a remote identity - RawMessage.get raise correctly - request body must be utf8 encoded correctly, do not use it for the moment ## [0.11.0] 2018-07-04 ### Added - client sign HTTP queries - API v1 and v2 check signed ecdsa http queries, but only log result - delete an user account - API for remote identities management - client can create an IMAP remote identities - poll remote identities to fetch an IMAP source - can use hashicorp vault to store sensible informations - fetch list of messages surrounding a given message - a go CLI to fix empty message participants ### Changed - optimization, prevent timeline to fetch multiple times the API - Internal email delivery is more reliable, with better error handling, reporting and logging. - scroll better on messages list ### Fixed - scrolls to specific message, and restore scroll positions when navigating between tabs. - change parent of a draft has no effects - sort messages of a discussion - discussion might not be up to date after creating a draft - sending two consecutives messages was failing when the second message is saved before clicking on the send button. - contact lookup can reference a deleted contact, don't fail ## [0.10.1] 2018-05-18 ### Fixed - a discussion_id should not be removed when patching a draft - bad device type mapping for smartphone devices, authentication wasn't possible - read message are flagged correctly read - search bar is more visible ## [0.10.0] 2018-05-14 ### Added - Confirmation is asked before deleting a message, a discussion or a contact - Messages have new computed property : `date_sort`. Messages' list is sorted on. - Basic support of new message's notifications - Poller and worker for IMAP remote identities fetch in backend ### Changed - in mobile view, draft form can toggle with an excerpt to not use half of the screen - Power to the not found unicorn ### Fixed - Make favicon accessible without authentication ## [0.9.2] 2018-04-12 ### Fixed - prevent button background color to change on hover when disabled - The draft delete button was always disabled even when draft was saved - The max file size was not rendered (attachments/contacts..) ## [0.9.1] 2018-04-05 ### Fixed - check if contact.title is null or undefined when displaying ContactBook ## [0.9.0] 2018-03-29 ### Added - Manage draft's attachements - Download message's attachements - Set context (safe, public, not safe) on signin (it has no effects yet) - Multiple messages delete on Timeline - Multiple messages' tags management on Timeline - Device management first part: declare new device with an ecdsa key - Backend notification base principle with a related API ### Changed - Refactor vcard parsing logic to get more informations - Contacts API are now v2 - Authenticate with a context and a device (known or a new one to declare) - Better email body sanitization - When updating a contact, compute it's new PI asynchronously ### Fixed - Explain that the NSA joke is a joke using a `:)` ## [0.8.1] 2018-01-25 ### Fixed - messages filtered by status draft/sent/received wasn't up-to-date - checkbox wasn't correctly checked in settings - fix BSOD (Black Screen Of Death) when an author is not in user's contact book ## [0.8.0] 2018-01-19 ### Added - Tags management: user's tags, tags on messages and contacts - add a `internal` tag to messages sent from the same instance than receiver ### Fixed - trim spaces in username on signin page - unable to load more messages on Timeline ## [0.7.0] 2017-12-22 ### Added - Timeline filter (all, received, sent, drafts) - Add a is_received flag on message structure - Add german translations - Add take a tour for current features ### Changed - Do not save the draft until body or participants is filled - Move activity spinner to the top right of contact page - Prevent double click on contact save ### Fixed - Signout on mobile doesn't disconnect - suggest participants fails if a contact has no emails - Max body size of a request (import contact) - Sort message by date sent for user's messages - Settings can be saved when display delay is changed - A draft still appears after been deleted - Show correctly the message in reply on the current draft - Notify when saving contact failed - Refresh contacts after a deletion - Get browsers' autofills on SigninForm ## [0.6.0] 2017-11-24 ### Added - Reset password - do not force phone nnumber normalization, accept everything and try to normalize - permit to set contact title on user input, do not compute it strictly ### Changed - User's contact cannot be deleted anymore - The name of the Timeline tab is now "Messages" instead of "Discussions" ### Fixed - Do not close the dropdown when receiving new suggestions - Disable send message when already sent and add visual feedback (spinner) - Disable import contact button on uploading and add a spinner - Save updated password strength after password modification - Set max file size for contact import - Efficient search highlights - Access to /user/security route - Contact's name consistency - On editing/creating contact, disable submit button if form is untouched ## [0.5.5] 2017-11-13 ### Added - Reset password API - In a draft, press comma or semicolon key to add a recipient - Basic search in messages and contacts - Delete a contact - Change password ### Fixed - In a draft to edit last recipient, pressing backspace does not remove last letter - In a draft, click outside of recipient list add a recipient - Unmarshal nested empty structures in go objects. - Save updated password strength after password modification ## [0.5.4] 2017-11-03 ### Fixed - unmarshal nested empty structures in go objects - ancestors_id always an array, even empty - save a draft notify correctly ## [0.5.3] 2017-11-02 ### Added - Delete a message from Timeline - Delete a draft - Notify the user the draft is saved after a manual save ### Fixed - Disable buttons send and save when draft is untouched - ancestors_id empty array and not null ## [0.5.2] 2017-10-31 ### Fixed - piwik site id environment variable ok with client build #590 - lower case local identity lookup #589 - empty string instead of none for family name #587 - plain text body unescaped only #586 - enforce uuid validation in apiv2 #584 ## [0.5.1] 2017-10-29 ### Fixed - fix sort in discussions - support https in api query configuration - Render correctly the frontend Server Side (%MARKUP% will not show up anymore) - fix empty UUID on patch #574 - lmtp crash with invalid nats message #575 - attach correctly to same discussion first outbound message and its reply #566 - handle better invalid message unmarshalling #579 ## [0.5.0] 2017-10-25 ### Added - create new contact - the excerpt of the message in reply into draft form - frontend custom settings for the instance and the running environment - API to allow users to change their password, with email notification - Compute first contact privacy features - Permit to restrict registration to a whitelist of user recovery emails ## [0.4.0] ### Added - Connect settings and apply - The brand new Timeline - Load more in the discussions - API for importance level messages filtering - API for full-text searches on messages & contacts - Compute importance level v0 for inbound messages ### Changed - Improve `PATCH` API - Backend produces `excerpt` for messages. - Backend produces plain or rich body using setting value - Frontend: refactoring Dropdown ### Fixed - Render tag list in contact book ## [0.3.0] 2017-08-31 ### Added - A python package caliopen_pi to group all logic related to privacy index compute - Add route `GET /v2/contacts/{contact_id}/identities` to search & retrieve identities from a contact. - New scene components related to Settings layout - New Importance Level range slider in tabs & alt navigation - In compose, add subject input field when recipient uses an email - Install postcss-loader and Autoprefixer (run by webpack) - Add user settings storage and API for management (GET and PATCH /settings) - Frontend can post draftID - Messages have 2 bodies : plain + HTML. - Only one body is returned to frontend for Message. - HTML body is sanitized before output to frontend. - MailMessages's subject decoded to always output an UTF-8 string. - Index operations return after index has been fully updated - API /v1/discussions returns messages ordered by last_messsage date_insert field. ### Changed - Rename layout and related scenes to - Display notification if contact update failed - support TAB for adding a participant to a discussion - use participant suggestions API for a new message - Handle 4xx errors when updating contact fails - Compose in multiple tabs messages, not only one. - Cross-browser `
API docs by Redocly

Caliopen HTTP/REST API (0.2.0)

Download OpenAPI specification:Download

users

Returns an auth token to build basicAuth for the p

Returns an auth token to build basicAuth for the provided credentials

Request Body schema: application/json
required
username
required
string
password
required
string
context
string
object (DefaultDevice)

Responses

Request samples

Content type
application/json
{
  • "username": "string",
  • "password": "string",
  • "context": "string",
  • "device": {
    }
}

Response samples

Content type
application/json
{
  • "username": "string",
  • "user_id": "string",
  • "tokens": {
    },
  • "device": {
    }
}

Gets `user + contact` objects for current logged-i

Gets user + contact objects for current logged-in user

Authorizations:
basicAuth
header Parameters
X-Caliopen-PI
required
string
Default: 1;100

The PI range requested in form of 1;100

Responses

Response samples

Content type
application/json
{
  • "contact": {
    },
  • "date_insert": "2019-08-24T14:15:22Z",
  • "family_name": "string",
  • "given_name": "string",
  • "name": "string",
  • "password": "string",
  • "params": { },
  • "privacy_features": { },
  • "pi": {
    },
  • "user_id": "string",
  • "recovery_email": "string"
}

Create a new User with provided credentials

Create a new User with provided credentials

Authorizations:
basicAuth
Request Body schema: application/json
required
object (NewContact)
username
required
string
password
required
string
recovery_email
required
string
tos
boolean
privacy
boolean
object (Settings)
object (DefaultDevice)

Responses

Request samples

Content type
application/json
{
  • "contact": {
    },
  • "username": "string",
  • "password": "string",
  • "recovery_email": "string",
  • "tos": true,
  • "privacy": true,
  • "settings": {
    },
  • "device": {
    }
}

Response samples

Content type
application/json
{
  • "location": "string"
}

Partially implemented. Currently only for changing

Partially implemented. Currently only for changing password.

Authorizations:
basicAuth
path Parameters
user_id
required
string
Request Body schema: application/json
required

the patch to apply. See 'Caliopen Patch RFC' within /doc directory.

required
object
Array of objects (Attachment)
body
string
body_is_plain
boolean
date
string <date-time>
date_delete
string <date-time>
date_insert
string <date-time>
date_sort
string <date-time>
discussion_id
string
object (ExternalReferences)
excerpt
string
user_identities
Array of strings
importance_level
integer <int32>
is_answered
boolean
is_draft
boolean
is_unread
boolean
is_received
boolean
message_id
string
parent_id
string
Array of objects (Participant)
privacy_features
object (PrivacyFeatures)
object (PI)
object (PIMessage)
raw_msg_id
string
subject
string
tags
Array of strings
protocol
string
user_id
string

Responses

Request samples

Content type
application/json
{
  • "current_state": {
    },
  • "attachments": [
    ],
  • "body": "string",
  • "body_is_plain": true,
  • "date": "2019-08-24T14:15:22Z",
  • "date_delete": "2019-08-24T14:15:22Z",
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_sort": "2019-08-24T14:15:22Z",
  • "discussion_id": "string",
  • "external_references": {
    },
  • "excerpt": "string",
  • "user_identities": [
    ],
  • "importance_level": 0,
  • "is_answered": true,
  • "is_draft": true,
  • "is_unread": true,
  • "is_received": true,
  • "message_id": "string",
  • "parent_id": "string",
  • "participants": [
    ],
  • "privacy_features": { },
  • "pi": {
    },
  • "pi_message": {
    },
  • "raw_msg_id": "string",
  • "subject": "string",
  • "tags": [
    ],
  • "protocol": "string",
  • "user_id": "string"
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

send an order to execute one (or many) action(s) r

send an order to execute one (or many) action(s) regarding the user : reset_password, etc. A successful execution of the action will probably modify one or more user's attribute(s) ## NOT YET IMPLEMENTED ##

Authorizations:
basicAuth
path Parameters
user_id
required
string
Request Body schema: application/json
required
actions
required
Array of strings
Items Enum: "send" "set_read" "set_unread" "reset_password" "delete" "device-validation"
params
object

Responses

Request samples

Content type
application/json
{
  • "actions": [
    ],
  • "params": { }
}

Response samples

Content type
application/json
{
  • "attachments": [
    ],
  • "body": "string",
  • "body_is_plain": true,
  • "date": "2019-08-24T14:15:22Z",
  • "date_delete": "2019-08-24T14:15:22Z",
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_sort": "2019-08-24T14:15:22Z",
  • "discussion_id": "string",
  • "external_references": {
    },
  • "excerpt": "string",
  • "user_identities": [
    ],
  • "importance_level": 0,
  • "is_answered": true,
  • "is_draft": true,
  • "is_unread": true,
  • "is_received": true,
  • "message_id": "string",
  • "parent_id": "string",
  • "participants": [
    ],
  • "privacy_features": { },
  • "pi": {
    },
  • "pi_message": {
    },
  • "raw_msg_id": "string",
  • "subject": "string",
  • "tags": [
    ],
  • "protocol": "string",
  • "user_id": "string"
}

Check if an username is available for creation wit

Check if an username is available for creation within Caliopen instance

query Parameters
username
required
string

Responses

Response samples

Content type
application/json
{
  • "username": "string",
  • "available": true
}

password

Partially implemented. Currently only for changing

Partially implemented. Currently only for changing password.

Authorizations:
basicAuth
path Parameters
user_id
required
string
Request Body schema: application/json
required

the patch to apply. See 'Caliopen Patch RFC' within /doc directory.

required
object
Array of objects (Attachment)
body
string
body_is_plain
boolean
date
string <date-time>
date_delete
string <date-time>
date_insert
string <date-time>
date_sort
string <date-time>
discussion_id
string
object (ExternalReferences)
excerpt
string
user_identities
Array of strings
importance_level
integer <int32>
is_answered
boolean
is_draft
boolean
is_unread
boolean
is_received
boolean
message_id
string
parent_id
string
Array of objects (Participant)
privacy_features
object (PrivacyFeatures)
object (PI)
object (PIMessage)
raw_msg_id
string
subject
string
tags
Array of strings
protocol
string
user_id
string

Responses

Request samples

Content type
application/json
{
  • "current_state": {
    },
  • "attachments": [
    ],
  • "body": "string",
  • "body_is_plain": true,
  • "date": "2019-08-24T14:15:22Z",
  • "date_delete": "2019-08-24T14:15:22Z",
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_sort": "2019-08-24T14:15:22Z",
  • "discussion_id": "string",
  • "external_references": {
    },
  • "excerpt": "string",
  • "user_identities": [
    ],
  • "importance_level": 0,
  • "is_answered": true,
  • "is_draft": true,
  • "is_unread": true,
  • "is_received": true,
  • "message_id": "string",
  • "parent_id": "string",
  • "participants": [
    ],
  • "privacy_features": { },
  • "pi": {
    },
  • "pi_message": {
    },
  • "raw_msg_id": "string",
  • "subject": "string",
  • "tags": [
    ],
  • "protocol": "string",
  • "user_id": "string"
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

send an order to execute one (or many) action(s) r

send an order to execute one (or many) action(s) regarding the user : reset_password, etc. A successful execution of the action will probably modify one or more user's attribute(s) ## NOT YET IMPLEMENTED ##

Authorizations:
basicAuth
path Parameters
user_id
required
string
Request Body schema: application/json
required
actions
required
Array of strings
Items Enum: "send" "set_read" "set_unread" "reset_password" "delete" "device-validation"
params
object

Responses

Request samples

Content type
application/json
{
  • "actions": [
    ],
  • "params": { }
}

Response samples

Content type
application/json
{
  • "attachments": [
    ],
  • "body": "string",
  • "body_is_plain": true,
  • "date": "2019-08-24T14:15:22Z",
  • "date_delete": "2019-08-24T14:15:22Z",
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_sort": "2019-08-24T14:15:22Z",
  • "discussion_id": "string",
  • "external_references": {
    },
  • "excerpt": "string",
  • "user_identities": [
    ],
  • "importance_level": 0,
  • "is_answered": true,
  • "is_draft": true,
  • "is_unread": true,
  • "is_received": true,
  • "message_id": "string",
  • "parent_id": "string",
  • "participants": [
    ],
  • "privacy_features": { },
  • "pi": {
    },
  • "pi_message": {
    },
  • "raw_msg_id": "string",
  • "subject": "string",
  • "tags": [
    ],
  • "protocol": "string",
  • "user_id": "string"
}

username

Check if an username is available for creation wit

Check if an username is available for creation within Caliopen instance

query Parameters
username
required
string

Responses

Response samples

Content type
application/json
{
  • "username": "string",
  • "available": true
}

settings

Returns settings belonging to current user

Returns settings belonging to current user

Authorizations:
basicAuth

Responses

Response samples

Content type
application/json
{
  • "default_locale": "fr-FR",
  • "message_display_format": "rich_text",
  • "contact_display_order": "given_name",
  • "contact_display_format": "family_name, given_name",
  • "notification_enabled": true,
  • "notification_message_preview": "always",
  • "notification_sound_enabled": false,
  • "notification_delay_disappear": 10
}

Update settings with rfc5789 and rfc7396 specifica

Update settings with rfc5789 and rfc7396 specifications

Authorizations:
basicAuth
Request Body schema: application/json
required

the patch to apply. See 'Caliopen Patch RFC' within /doc directory.

object
default_locale
string
Default: "fr-FR"
message_display_format
string
Default: "rich_text"
contact_display_order
string
Default: "given_name"
contact_display_format
string
Default: "family_name, given_name"
notification_enabled
boolean
Default: true
notification_message_preview
string
Default: "always"
notification_sound_enabled
boolean
Default: false
notification_delay_disappear
integer
Default: 10

Responses

Request samples

Content type
application/json
{
  • "current_state": {
    },
  • "default_locale": "fr-FR",
  • "message_display_format": "rich_text",
  • "contact_display_order": "given_name",
  • "contact_display_format": "family_name, given_name",
  • "notification_enabled": true,
  • "notification_message_preview": "always",
  • "notification_sound_enabled": false,
  • "notification_delay_disappear": 10
}

identities

returns the list of user's local identities

returns the list of user's local identities

Authorizations:
basicAuth

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "local_identities": [
    ]
}

returns the list of user's remote identities, or f

returns the list of user's remote identities, or filtered if query param

Authorizations:
basicAuth
query Parameters
pending
string

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "remote_identities": [
    ]
}

create a new remote identity for user

create a new remote identity for user

Authorizations:
basicAuth
Request Body schema: application/json
required
object
display_name
string
identifier
required
string
object
protocol
required
string
Enum: "email" "twitter"
status
string
Enum: "active" "inactive" "deleted"
type
string
Enum: "local" "remote"
user_id
string

Responses

Request samples

Content type
application/json
{
  • "credentials": {
    },
  • "display_name": "string",
  • "identifier": "string",
  • "infos": {
    },
  • "protocol": "email",
  • "status": "active",
  • "type": "local",
  • "user_id": "string"
}

Response samples

Content type
application/json
{
  • "location": "string",
  • "identifier": "string"
}

returns a remote identity belonging to user

returns a remote identity belonging to user

Authorizations:
basicAuth
path Parameters
identifier
required
string

Responses

Response samples

Content type
application/json
{
  • "credentials": {
    },
  • "display_name": "string",
  • "identity_id": "string",
  • "identifier": "string",
  • "infos": {
    },
  • "last_check": "2019-08-24T14:15:22Z",
  • "protocol": "email",
  • "status": "active",
  • "type": "local",
  • "user_id": "string"
}

Delete a remote identity belonging to user

Delete a remote identity belonging to user

Authorizations:
basicAuth
path Parameters
identifier
required
string

Responses

update a remote identity with rfc5789 and rfc7396

update a remote identity with rfc5789 and rfc7396 specifications

Authorizations:
basicAuth
path Parameters
identifier
required
string
Request Body schema: application/json
required

the patch to apply. See 'Caliopen Patch RFC' within /doc directory.

required
object
object
display_name
string
identity_id
string
identifier
string
object
last_check
string <date-time>
protocol
string
Enum: "email" "twitter"
status
string
Enum: "active" "inactive" "deleted"
type
string
Enum: "local" "remote"
user_id
string

Responses

Request samples

Content type
application/json
{
  • "current_state": {
    },
  • "credentials": {
    },
  • "display_name": "string",
  • "identity_id": "string",
  • "identifier": "string",
  • "infos": {
    },
  • "last_check": "2019-08-24T14:15:22Z",
  • "protocol": "email",
  • "status": "active",
  • "type": "local",
  • "user_id": "string"
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

returns list of supported external providers for r

returns list of supported external providers for remote identities creation

Authorizations:
basicAuth

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "providers": [
    ]
}

returns provider's properties, notably the url to

returns provider's properties, notably the url to call for initiating Oauth process

Authorizations:
basicAuth
path Parameters
provider_name
required
string
query Parameters
identifier
string

Responses

Response samples

Content type
application/json
{
  • "name": "string",
  • "oauth_request_url": "string",
  • "protocol": "string"
}

url registered at provider to which its API will r

url registered at provider to which its API will redirect user after user authentication

path Parameters
provider_name
required
string

Responses

Response samples

Content type
application/json
{
  • "success": true,
  • "message": "string"
}

passwords

Route to receive a "reset password" request from a

Route to receive a "reset password" request from an anonymous user.

Request Body schema: application/json
required

data the user has typed into the reset form

recovery_email
string
username
string

Responses

Request samples

Content type
application/json
{
  • "recovery_email": "string",
  • "username": "string"
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

Returns an auth token to build basicAuth, if the t

Returns an auth token to build basicAuth, if the token in path is valid

path Parameters
token
required
string

Responses

Response samples

Content type
application/json
{
  • "error": {
    }
}

User posts a new password

User posts a new password

path Parameters
token
required
string
Request Body schema: application/json
required

A simple json with the new password as a string

password
required
string

Responses

Request samples

Content type
application/json
{
  • "password": "string"
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

providers

returns list of supported external providers for r

returns list of supported external providers for remote identities creation

Authorizations:
basicAuth

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "providers": [
    ]
}

returns provider's properties, notably the url to

returns provider's properties, notably the url to call for initiating Oauth process

Authorizations:
basicAuth
path Parameters
provider_name
required
string
query Parameters
identifier
string

Responses

Response samples

Content type
application/json
{
  • "name": "string",
  • "oauth_request_url": "string",
  • "protocol": "string"
}

url registered at provider to which its API will r

url registered at provider to which its API will redirect user after user authentication

path Parameters
provider_name
required
string

Responses

Response samples

Content type
application/json
{
  • "success": true,
  • "message": "string"
}

serves an index to test Oauth processes

serves an index to test Oauth processes

Responses

contacts

Returns contacts list for current user according t

Returns contacts list for current user according to given params

Authorizations:
basicAuth
query Parameters
limit
integer

number of contacts to return per page

offset
integer

number of pages to skip from the response

uri
string

return contact that has this uri embedded, if any

header Parameters
X-Caliopen-PI
required
string
Default: 1;100

The PI range requested in form of 1;100

X-Caliopen-IL
required
string
Default: -10;10

The Importance Level range requested in form of -10;10

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "contacts": [
    ]
}

Create a new contact for the logged-in user

Create a new contact for the logged-in user

Authorizations:
basicAuth
Request Body schema: application/json

the contact to create

additional_name
string
Array of objects (PostalAddress)
avatar
string
Array of objects (NewEmail)
family_name
string
given_name
string
title
string
groups
Array of strings
Array of objects (SocialIdentity)
ims
Array of objects
infos
object
name_prefix
string
name_suffix
string
organizations
Array of objects
Array of objects (Phone)
Array of objects (PublicKey)

Responses

Request samples

Content type
application/json
{
  • "additional_name": "string",
  • "addresses": [
    ],
  • "avatar": "string",
  • "emails": [
    ],
  • "family_name": "string",
  • "given_name": "string",
  • "title": "string",
  • "groups": [
    ],
  • "identities": [
    ],
  • "ims": [
    ],
  • "infos": { },
  • "name_prefix": "string",
  • "name_suffix": "string",
  • "organizations": [
    ],
  • "phones": [
    ],
  • "public_keys": [
    ]
}

Response samples

Content type
application/json
{
  • "location": "string",
  • "contact_id": "string"
}

Returns a contact

Returns a contact

Authorizations:
basicAuth
path Parameters
contact_id
required
string

Responses

Response samples

Content type
application/json
{
  • "additional_name": "string",
  • "addresses": [
    ],
  • "avatar": "string",
  • "contact_id": "string",
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_update": "2019-08-24T14:15:22Z",
  • "deleted": "2019-08-24T14:15:22Z",
  • "emails": [
    ],
  • "family_name": "string",
  • "given_name": "string",
  • "groups": [
    ],
  • "identities": [
    ],
  • "ims": [
    ],
  • "infos": { },
  • "name_prefix": "string",
  • "name_suffix": "string",
  • "organizations": [
    ],
  • "phones": [
    ],
  • "pi": {
    },
  • "privacy_features": { },
  • "public_keys": [
    ],
  • "tags": [
    ],
  • "title": "string",
  • "user_id": "string"
}

Delete a contact

Delete a contact

Authorizations:
basicAuth
path Parameters
contact_id
required
string

Responses

update a contact with rfc5789 and rfc7396 specific

update a contact with rfc5789 and rfc7396 specifications

Authorizations:
basicAuth
path Parameters
contact_id
required
string
Request Body schema: application/json
required

the patch to apply. See 'Caliopen Patch RFC' within /doc directory.

required
object
additional_name
string
Array of objects (PostalAddress)
avatar
string
contact_id
string
date_insert
string <date-time>
date_update
string <date-time>
deleted
string <date-time>
Array of objects (Email)
family_name
string
given_name
string
groups
Array of strings
Array of objects (SocialIdentity)
Array of objects (IM)
infos
object
name_prefix
string
name_suffix
string
Array of objects (Organization)
Array of objects (Phone)
object (PI)
privacy_features
object (PrivacyFeatures)
Array of objects (PublicKey)
tags
Array of strings
title
string
user_id
string

Responses

Request samples

Content type
application/json
{
  • "current_state": {
    },
  • "additional_name": "string",
  • "addresses": [
    ],
  • "avatar": "string",
  • "contact_id": "string",
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_update": "2019-08-24T14:15:22Z",
  • "deleted": "2019-08-24T14:15:22Z",
  • "emails": [
    ],
  • "family_name": "string",
  • "given_name": "string",
  • "groups": [
    ],
  • "identities": [
    ],
  • "ims": [
    ],
  • "infos": { },
  • "name_prefix": "string",
  • "name_suffix": "string",
  • "organizations": [
    ],
  • "phones": [
    ],
  • "pi": {
    },
  • "privacy_features": { },
  • "public_keys": [
    ],
  • "tags": [
    ],
  • "title": "string",
  • "user_id": "string"
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

returns a list of contact's identities

returns a list of contact's identities

Authorizations:
basicAuth
path Parameters
contact_id
required
string

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "contact_identities": [
    ]
}

update tags list for contact

update tags list for contact

Authorizations:
basicAuth
path Parameters
contact_id
required
string
Request Body schema: application/json
required

the patch to apply. See 'Caliopen Patch RFC' within /doc directory.

required
object
additional_name
string
Array of objects (PostalAddress)
avatar
string
contact_id
string
date_insert
string <date-time>
date_update
string <date-time>
deleted
string <date-time>
Array of objects (Email)
family_name
string
given_name
string
groups
Array of strings
Array of objects (SocialIdentity)
Array of objects (IM)
infos
object
name_prefix
string
name_suffix
string
Array of objects (Organization)
Array of objects (Phone)
object (PI)
privacy_features
object (PrivacyFeatures)
Array of objects (PublicKey)
tags
Array of strings
title
string
user_id
string

Responses

Request samples

Content type
application/json
{
  • "current_state": {
    },
  • "additional_name": "string",
  • "addresses": [
    ],
  • "avatar": "string",
  • "contact_id": "string",
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_update": "2019-08-24T14:15:22Z",
  • "deleted": "2019-08-24T14:15:22Z",
  • "emails": [
    ],
  • "family_name": "string",
  • "given_name": "string",
  • "groups": [
    ],
  • "identities": [
    ],
  • "ims": [
    ],
  • "infos": { },
  • "name_prefix": "string",
  • "name_suffix": "string",
  • "organizations": [
    ],
  • "phones": [
    ],
  • "pi": {
    },
  • "privacy_features": { },
  • "public_keys": [
    ],
  • "tags": [
    ],
  • "title": "string",
  • "user_id": "string"
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

update tags list for contact

update tags list for contact

Authorizations:
basicAuth
path Parameters
contact_id
required
string
Request Body schema: application/json
required

the patch to apply. See 'Caliopen Patch RFC' within /doc directory.

tags
required
Array of strings
required
object

Responses

Request samples

Content type
application/json
{
  • "tags": [
    ],
  • "current_state": {
    }
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

Simple API to execute full-text searches within us

Simple API to execute full-text searches within user's indexes. A more complexe API will be available with a POST verb.

Authorizations:
basicAuth
query Parameters
term
required
string >= 3 characters

the search string

field
string

name of a field on which to perform the search. If omitted defaults to « _all ».

doctype
string
Enum: "message" "contact" ""

type of documents to narrow the search to.

limit
integer

number of documents to return per page, but only if param «type» is present.

offset
integer

number of pages to skip from the response, but only if param «type» is present.

header Parameters
X-Caliopen-IL
required
string
Default: -10;10

The Importance Level range requested in form of -10;10

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "message_hits": {
    },
  • "contact_hits": {
    }
}

Not yet implemented. Future route for more complex

Not yet implemented. Future route for more complexe searches.

Authorizations:
basicAuth

Responses

tags

update tags list for contact

update tags list for contact

Authorizations:
basicAuth
path Parameters
contact_id
required
string
Request Body schema: application/json
required

the patch to apply. See 'Caliopen Patch RFC' within /doc directory.

required
object
additional_name
string
Array of objects (PostalAddress)
avatar
string
contact_id
string
date_insert
string <date-time>
date_update
string <date-time>
deleted
string <date-time>
Array of objects (Email)
family_name
string
given_name
string
groups
Array of strings
Array of objects (SocialIdentity)
Array of objects (IM)
infos
object
name_prefix
string
name_suffix
string
Array of objects (Organization)
Array of objects (Phone)
object (PI)
privacy_features
object (PrivacyFeatures)
Array of objects (PublicKey)
tags
Array of strings
title
string
user_id
string

Responses

Request samples

Content type
application/json
{
  • "current_state": {
    },
  • "additional_name": "string",
  • "addresses": [
    ],
  • "avatar": "string",
  • "contact_id": "string",
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_update": "2019-08-24T14:15:22Z",
  • "deleted": "2019-08-24T14:15:22Z",
  • "emails": [
    ],
  • "family_name": "string",
  • "given_name": "string",
  • "groups": [
    ],
  • "identities": [
    ],
  • "ims": [
    ],
  • "infos": { },
  • "name_prefix": "string",
  • "name_suffix": "string",
  • "organizations": [
    ],
  • "phones": [
    ],
  • "pi": {
    },
  • "privacy_features": { },
  • "public_keys": [
    ],
  • "tags": [
    ],
  • "title": "string",
  • "user_id": "string"
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

update tags list for contact

update tags list for contact

Authorizations:
basicAuth
path Parameters
contact_id
required
string
Request Body schema: application/json
required

the patch to apply. See 'Caliopen Patch RFC' within /doc directory.

tags
required
Array of strings
required
object

Responses

Request samples

Content type
application/json
{
  • "tags": [
    ],
  • "current_state": {
    }
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

update tags list for message

update tags list for message

Authorizations:
basicAuth
path Parameters
message_id
required
string
Request Body schema: application/json
required

the patch to apply. See 'Caliopen Patch RFC' within /doc directory.

tags
required
Array of strings
required
object

Responses

Request samples

Content type
application/json
{
  • "tags": [
    ],
  • "current_state": {
    }
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

Returns tags visible to current user according to

Returns tags visible to current user according to given parameters

Authorizations:
basicAuth

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "tags": [
    ]
}

Create a new Tag for an user

Create a new Tag for an user

Authorizations:
basicAuth
Request Body schema: application/json
required
label
required
string
importance_level
integer <int32>

Responses

Request samples

Content type
application/json
{
  • "label": "string",
  • "importance_level": 0
}

Response samples

Content type
application/json
{
  • "location": "string"
}

Retrieve tag infos

Retrieve tag infos

Authorizations:
basicAuth
path Parameters
tag_id
required
string

Responses

Response samples

Content type
application/json
{
  • "date_insert": "2019-08-24T14:15:22Z",
  • "type": "user",
  • "label": "string",
  • "importance_level": 0
}

update a tag

update a tag

Authorizations:
basicAuth
path Parameters
tag_id
required
string
Request Body schema: application/json
required

the patch to apply. See 'Caliopen Patch RFC' within /doc directory.

label
string
importance_level
integer <int32>
required
object

Responses

Request samples

Content type
application/json
{
  • "label": "string",
  • "importance_level": 0,
  • "current_state": {
    }
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

Delete a tag belonging to an user

Delete a tag belonging to an user

Authorizations:
basicAuth
path Parameters
tag_id
required
string

Responses

pgp

Add a pgp public key to a contact

Add a pgp public key to a contact

Authorizations:
basicAuth
path Parameters
contact_id
required
string
Request Body schema: application/json
required
key
required
string

DER or PEM key, base64 encoded

label
required
string

Responses

Request samples

Content type
application/json
{
  • "key": "string",
  • "label": "string"
}

Response samples

Content type
application/json
{
  • "location": "string",
  • "publickey_id": "string"
}

Returns all publickeys linked to contact

Returns all publickeys linked to contact

Authorizations:
basicAuth
path Parameters
contact_id
required
string

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "pubkeys": [
    ]
}

Retrieve publickey

Retrieve publickey

Authorizations:
basicAuth
path Parameters
contact_id
required
string
pubkey_id
required
string

Responses

Response samples

Content type
application/json
{
  • "alg": "string",
  • "crv": "string",
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_update": "2019-08-24T14:15:22Z",
  • "emails": [
    ],
  • "expire_date": "2019-08-24T14:15:22Z",
  • "fingerprint": "string",
  • "kty": "string",
  • "key_id": "string",
  • "type": "string",
  • "resource_id": "string",
  • "resource_type": "string",
  • "size": 0,
  • "use": "string",
  • "user_id": "string",
  • "x": 0,
  • "y": 0,
  • "key": "string",
  • "label": "string"
}

update a public key

update a public key

Authorizations:
basicAuth
path Parameters
contact_id
required
string
pubkey_id
required
string
Request Body schema: application/json
required

the patch to apply. Property label is the only one patchable.

label
required
string
required
object

Responses

Request samples

Content type
application/json
{
  • "label": "string",
  • "current_state": {
    }
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

Delete a public key

Delete a public key

Authorizations:
basicAuth
path Parameters
contact_id
required
string
pubkey_id
required
string

Responses

keys

Add a pgp public key to a contact

Add a pgp public key to a contact

Authorizations:
basicAuth
path Parameters
contact_id
required
string
Request Body schema: application/json
required
key
required
string

DER or PEM key, base64 encoded

label
required
string

Responses

Request samples

Content type
application/json
{
  • "key": "string",
  • "label": "string"
}

Response samples

Content type
application/json
{
  • "location": "string",
  • "publickey_id": "string"
}

Returns all publickeys linked to contact

Returns all publickeys linked to contact

Authorizations:
basicAuth
path Parameters
contact_id
required
string

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "pubkeys": [
    ]
}

Retrieve publickey

Retrieve publickey

Authorizations:
basicAuth
path Parameters
contact_id
required
string
pubkey_id
required
string

Responses

Response samples

Content type
application/json
{
  • "alg": "string",
  • "crv": "string",
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_update": "2019-08-24T14:15:22Z",
  • "emails": [
    ],
  • "expire_date": "2019-08-24T14:15:22Z",
  • "fingerprint": "string",
  • "kty": "string",
  • "key_id": "string",
  • "type": "string",
  • "resource_id": "string",
  • "resource_type": "string",
  • "size": 0,
  • "use": "string",
  • "user_id": "string",
  • "x": 0,
  • "y": 0,
  • "key": "string",
  • "label": "string"
}

update a public key

update a public key

Authorizations:
basicAuth
path Parameters
contact_id
required
string
pubkey_id
required
string
Request Body schema: application/json
required

the patch to apply. Property label is the only one patchable.

label
required
string
required
object

Responses

Request samples

Content type
application/json
{
  • "label": "string",
  • "current_state": {
    }
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

Delete a public key

Delete a public key

Authorizations:
basicAuth
path Parameters
contact_id
required
string
pubkey_id
required
string

Responses

/v2/imports

Authorizations:
basicAuth
Request Body schema: multipart/form-data
required
file
required
string <binary>

the vcard file to upload

Responses

Response samples

Content type
application/json
{
  • "error": {
    }
}

discussions

Returns the list of discussions for current user a

Returns the list of discussions for current user according to given filter

Authorizations:
basicAuth
query Parameters
limit
integer

number of discussions to return per page

offset
integer

number of discussions to skip for pagination

header Parameters
X-Caliopen-PI
required
string
Default: 0;100

The PI range requested in form of 0;100

X-Caliopen-IL
required
string
Default: -10;10

The Importance Level range requested in form of -10;10

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "discussions": [
    ]
}

Returns metadata of a discussion

Returns metadata of a discussion

Authorizations:
basicAuth
path Parameters
discussion_id
required
string

Responses

Response samples

Content type
application/json
{
  • "attachment_count": 0,
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_update": "2019-08-24T14:15:22Z",
  • "discussion_id": "string",
  • "importance_level": 0,
  • "participants": [
    ],
  • "tags": [
    ],
  • "excerpt": "string",
  • "subject": "string",
  • "protocol": "string",
  • "total_count": 0,
  • "unread_count": 0,
  • "aliases": [
    ],
  • "last_message_id": "string",
  • "last_message_date": "2019-08-24T14:15:22Z",
  • "last_message_subject": "string"
}

messages

Create a new Message (draft) for an user

Create a new Message (draft) for an user

Authorizations:
basicAuth
Request Body schema: application/json
required
Array of objects (Attachment)
body
string
discussion_id
string
user_identities
required
Array of strings
message_id
string
parent_id
string
Array of objects (Participant)
subject
string
privacy_features
object (PrivacyFeatures)

Responses

Request samples

Content type
application/json
{
  • "attachments": [
    ],
  • "body": "string",
  • "discussion_id": "string",
  • "user_identities": [
    ],
  • "message_id": "string",
  • "parent_id": "string",
  • "participants": [
    ],
  • "subject": "string",
  • "privacy_features": { }
}

Response samples

Content type
application/json
{
  • "location": "string"
}

Returns the list of messages for current user acco

Returns the list of messages for current user according to given parameters/filter

Authorizations:
basicAuth
query Parameters
discussion_id
string

filter messages belonging to a specific discussion

limit
integer

number of messages to return per page

offset
integer

number of pages to skip from the response

msg_id
string

if provided with range[] param, specify a message_id around which messages will be fetched

range[]
string

boundaries param if message_id param is provided [before, after]

header Parameters
X-Caliopen-PI
required
string
Default: 0;100

The PI range requested in form of 0;100

X-Caliopen-IL
required
string
Default: -10;10

The Importance Level range requested in form of -10;10

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "messages": [
    ]
}

update a draft with rfc5789 and rfc7396 specificat

update a draft with rfc5789 and rfc7396 specifications

Authorizations:
basicAuth
path Parameters
message_id
required
string
Request Body schema: application/json
required

the patch to apply. See 'Caliopen Patch RFC' within /doc directory.

required
object
Array of objects (Attachment)
body
string
body_is_plain
boolean
date
string <date-time>
date_delete
string <date-time>
date_insert
string <date-time>
date_sort
string <date-time>
discussion_id
string
object (ExternalReferences)
excerpt
string
user_identities
Array of strings
importance_level
integer <int32>
is_answered
boolean
is_draft
boolean
is_unread
boolean
is_received
boolean
message_id
string
parent_id
string
Array of objects (Participant)
privacy_features
object (PrivacyFeatures)
object (PI)
object (PIMessage)
raw_msg_id
string
subject
string
tags
Array of strings
protocol
string
user_id
string

Responses

Request samples

Content type
application/json
{
  • "current_state": {
    },
  • "attachments": [
    ],
  • "body": "string",
  • "body_is_plain": true,
  • "date": "2019-08-24T14:15:22Z",
  • "date_delete": "2019-08-24T14:15:22Z",
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_sort": "2019-08-24T14:15:22Z",
  • "discussion_id": "string",
  • "external_references": {
    },
  • "excerpt": "string",
  • "user_identities": [
    ],
  • "importance_level": 0,
  • "is_answered": true,
  • "is_draft": true,
  • "is_unread": true,
  • "is_received": true,
  • "message_id": "string",
  • "parent_id": "string",
  • "participants": [
    ],
  • "privacy_features": { },
  • "pi": {
    },
  • "pi_message": {
    },
  • "raw_msg_id": "string",
  • "subject": "string",
  • "tags": [
    ],
  • "protocol": "string",
  • "user_id": "string"
}

Delete a message belonging to an user

Delete a message belonging to an user

Authorizations:
basicAuth
path Parameters
message_id
required
string

Responses

returns a message

returns a message

Authorizations:
basicAuth
path Parameters
message_id
required
string

Responses

Response samples

Content type
application/json
{
  • "attachments": [
    ],
  • "body": "string",
  • "body_is_plain": true,
  • "date": "2019-08-24T14:15:22Z",
  • "date_delete": "2019-08-24T14:15:22Z",
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_sort": "2019-08-24T14:15:22Z",
  • "discussion_id": "string",
  • "external_references": {
    },
  • "excerpt": "string",
  • "user_identities": [
    ],
  • "importance_level": 0,
  • "is_answered": true,
  • "is_draft": true,
  • "is_unread": true,
  • "is_received": true,
  • "message_id": "string",
  • "parent_id": "string",
  • "participants": [
    ],
  • "privacy_features": { },
  • "pi": {
    },
  • "pi_message": {
    },
  • "raw_msg_id": "string",
  • "subject": "string",
  • "tags": [
    ],
  • "protocol": "string",
  • "user_id": "string"
}

update tags list for message

update tags list for message

Authorizations:
basicAuth
path Parameters
message_id
required
string
Request Body schema: application/json
required

the patch to apply. See 'Caliopen Patch RFC' within /doc directory.

tags
required
Array of strings
required
object

Responses

Request samples

Content type
application/json
{
  • "tags": [
    ],
  • "current_state": {
    }
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

send an order to execute one (or many) action(s) f

send an order to execute one (or many) action(s) for the given message : send, etc. A successful execution of the action will probably modify one or more message's attribute(s)

Authorizations:
basicAuth
path Parameters
message_id
required
string
Request Body schema: application/json
required
actions
required
Array of strings
Items Enum: "send" "set_read" "set_unread" "reset_password" "delete" "device-validation"
params
object

Responses

Request samples

Content type
application/json
{
  • "actions": [
    ],
  • "params": { }
}

Response samples

Content type
application/json
{
  • "attachments": [
    ],
  • "body": "string",
  • "body_is_plain": true,
  • "date": "2019-08-24T14:15:22Z",
  • "date_delete": "2019-08-24T14:15:22Z",
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_sort": "2019-08-24T14:15:22Z",
  • "discussion_id": "string",
  • "external_references": {
    },
  • "excerpt": "string",
  • "user_identities": [
    ],
  • "importance_level": 0,
  • "is_answered": true,
  • "is_draft": true,
  • "is_unread": true,
  • "is_received": true,
  • "message_id": "string",
  • "parent_id": "string",
  • "participants": [
    ],
  • "privacy_features": { },
  • "pi": {
    },
  • "pi_message": {
    },
  • "raw_msg_id": "string",
  • "subject": "string",
  • "tags": [
    ],
  • "protocol": "string",
  • "user_id": "string"
}

(for draft only) upload a file to server and add a

(for draft only) upload a file to server and add attachment reference to the draft.

Authorizations:
basicAuth
path Parameters
message_id
required
string
Request Body schema: multipart/form-data
required
attachment
required
string <binary>

the attachment file to upload

Responses

Response samples

Content type
application/json
{
  • "temp_id": "string"
}

Download file from server

Download file from server

Authorizations:
basicAuth
path Parameters
message_id
required
string
attachment_id
required
string

attachment position within message

Responses

(for drafts only) delete temporary file and remove

(for drafts only) delete temporary file and remove attachment reference from the draft.

Authorizations:
basicAuth
path Parameters
message_id
required
string
attachment_id
required
string

attachment's temporary id.

Responses

Returns a raw message

Returns a raw message

Authorizations:
basicAuth
path Parameters
raw_msg_id
required
string

Responses

Simple API to execute full-text searches within us

Simple API to execute full-text searches within user's indexes. A more complexe API will be available with a POST verb.

Authorizations:
basicAuth
query Parameters
term
required
string >= 3 characters

the search string

field
string

name of a field on which to perform the search. If omitted defaults to « _all ».

doctype
string
Enum: "message" "contact" ""

type of documents to narrow the search to.

limit
integer

number of documents to return per page, but only if param «type» is present.

offset
integer

number of pages to skip from the response, but only if param «type» is present.

header Parameters
X-Caliopen-IL
required
string
Default: -10;10

The Importance Level range requested in form of -10;10

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "message_hits": {
    },
  • "contact_hits": {
    }
}

Not yet implemented. Future route for more complex

Not yet implemented. Future route for more complexe searches.

Authorizations:
basicAuth

Responses

attachments

(for draft only) upload a file to server and add a

(for draft only) upload a file to server and add attachment reference to the draft.

Authorizations:
basicAuth
path Parameters
message_id
required
string
Request Body schema: multipart/form-data
required
attachment
required
string <binary>

the attachment file to upload

Responses

Response samples

Content type
application/json
{
  • "temp_id": "string"
}

Download file from server

Download file from server

Authorizations:
basicAuth
path Parameters
message_id
required
string
attachment_id
required
string

attachment position within message

Responses

(for drafts only) delete temporary file and remove

(for drafts only) delete temporary file and remove attachment reference from the draft.

Authorizations:
basicAuth
path Parameters
message_id
required
string
attachment_id
required
string

attachment's temporary id.

Responses

devices

Returns devices belonging to current user accordin

Returns devices belonging to current user according to given parameters

Authorizations:
basicAuth
query Parameters
limit
integer

number of devices to return per page

offset
integer

number of pages to skip from the response

header Parameters
X-Caliopen-PI
required
string
Default: 1;100

The PI range requested in form of 1;100

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "devices": [
    ]
}

Returns a device

Returns a device

Authorizations:
basicAuth
path Parameters
device_id
required
string

Responses

Response samples

Content type
application/json
{
  • "device_id": "string",
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_revoked": "2019-08-24T14:15:22Z",
  • "ip_creation": "string",
  • "name": "string",
  • "privacy_features": { },
  • "pi": {
    },
  • "status": "string",
  • "type": "string",
  • "user_id": "string",
  • "user_agent": "string",
  • "locations": [
    ],
  • "public_keys": [
    ]
}

Delete a device

Delete a device

Authorizations:
basicAuth
path Parameters
device_id
required
string

Responses

update a device with rfc5789 and rfc7396 specifica

update a device with rfc5789 and rfc7396 specifications

Authorizations:
basicAuth
path Parameters
device_id
required
string
Request Body schema: application/json
required

the patch to apply. See 'Caliopen Patch RFC' within /doc directory.

required
object
device_id
string
ip_creation
string

ip address at creation

Array of objects (DeviceLocation)
name
string
Array of objects (PublicKey)
status
string
type
string
Default: "unknow"
Enum: "other" "desktop" "laptop" "smartphone" "tablet"
user_agent
string

Responses

Request samples

Content type
application/json
{
  • "current_state": {
    },
  • "device_id": "string",
  • "ip_creation": "string",
  • "locations": [
    ],
  • "name": "string",
  • "public_keys": [
    ],
  • "status": "string",
  • "type": "other",
  • "user_agent": "string"
}

Response samples

Content type
application/json
{
  • "error": {
    }
}

Route to receive orders to trigger actions on a de

Route to receive orders to trigger actions on a device

Authorizations:
basicAuth
path Parameters
device_id
required
string
Request Body schema: application/json
required
actions
required
Array of strings
Items Enum: "send" "set_read" "set_unread" "reset_password" "delete" "device-validation"
params
object

Responses

Request samples

Content type
application/json
{
  • "actions": [
    ],
  • "params": { }
}

Response samples

Content type
application/json
{
  • "attachments": [
    ],
  • "body": "string",
  • "body_is_plain": true,
  • "date": "2019-08-24T14:15:22Z",
  • "date_delete": "2019-08-24T14:15:22Z",
  • "date_insert": "2019-08-24T14:15:22Z",
  • "date_sort": "2019-08-24T14:15:22Z",
  • "discussion_id": "string",
  • "external_references": {
    },
  • "excerpt": "string",
  • "user_identities": [
    ],
  • "importance_level": 0,
  • "is_answered": true,
  • "is_draft": true,
  • "is_unread": true,
  • "is_received": true,
  • "message_id": "string",
  • "parent_id": "string",
  • "participants": [
    ],
  • "privacy_features": { },
  • "pi": {
    },
  • "pi_message": {
    },
  • "raw_msg_id": "string",
  • "subject": "string",
  • "tags": [
    ],
  • "protocol": "string",
  • "user_id": "string"
}

complete device validation if token is valid

complete device validation if token is valid

Authorizations:
basicAuth
path Parameters
token
required
string

Responses

Response samples

Content type
application/json
{
  • "error": {
    }
}

participants

Returns a list of suggestions according to given p

Returns a list of suggestions according to given parameters/filter. Search is performed within current user's indexes (messages & contacts).

Authorizations:
basicAuth
query Parameters
context
required
string
Value: "msg_compose"

current user's context (to optimize suggest relevance)

q
required
string >= 3 characters

a string (3 chars at least) from which to perform the suggestion search

Responses

Response samples

Content type
application/json
[
  • {
    }
]

Returns discussion related to a list of participan

Returns discussion related to a list of participants

Authorizations:
basicAuth
Request Body schema: application/json
required
Array
address
required
string
contact_ids
Array of strings
label
string
protocol
required
string
type
required
string
Enum: "To" "Cc" "Bcc" "From" "Reply-To" "Sender"

Responses

Request samples

Content type
application/json
[
  • {
    }
]

Response samples

Content type
application/json
{
  • "hash": "string",
  • "discussion_id": "string"
}

suggest

Returns a list of suggestions according to given p

Returns a list of suggestions according to given parameters/filter. Search is performed within current user's indexes (messages & contacts).

Authorizations:
basicAuth
query Parameters
context
required
string
Value: "msg_compose"

current user's context (to optimize suggest relevance)

q
required
string >= 3 characters

a string (3 chars at least) from which to perform the suggestion search

Responses

Response samples

Content type
application/json
[
  • {
    }
]

search

Simple API to execute full-text searches within us

Simple API to execute full-text searches within user's indexes. A more complexe API will be available with a POST verb.

Authorizations:
basicAuth
query Parameters
term
required
string >= 3 characters

the search string

field
string

name of a field on which to perform the search. If omitted defaults to « _all ».

doctype
string
Enum: "message" "contact" ""

type of documents to narrow the search to.

limit
integer

number of documents to return per page, but only if param «type» is present.

offset
integer

number of pages to skip from the response, but only if param «type» is present.

header Parameters
X-Caliopen-IL
required
string
Default: -10;10

The Importance Level range requested in form of -10;10

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "message_hits": {
    },
  • "contact_hits": {
    }
}

Not yet implemented. Future route for more complex

Not yet implemented. Future route for more complexe searches.

Authorizations:
basicAuth

Responses

notifications

Returns pending notifications

Returns pending notifications

Authorizations:
basicAuth
query Parameters
from
string

oldest timestamp or uuid to retrieve (older notifications will not be fetched). RFC3339 format if time, UUIDv1 if id.

to
string

earlest timestamp or uuid to retrieve (earler notifications will not be fetched). RFC3339 format if time, UUIDv1 if id.

Responses

Response samples

Content type
application/json
{
  • "total": 0,
  • "notifications": [
    ]
}

Delete pending notifications by time range

Delete pending notifications by time range

Authorizations:
basicAuth
query Parameters
until
string <date-time>

delete all notifications with a timestamp before until time (RFC3339 format)

Responses

Response samples

Content type
application/json
{
  • "error": {
    }
}

Returns a notification

Returns a notification

Authorizations:
basicAuth
path Parameters
notification_id
required
string

Responses

Response samples

Content type
application/json
{
  • "body": "string",
  • "emitter": "string",
  • "notif_id": "string",
  • "reference": "string",
  • "type": "string",
  • "user_id": "string",
  • "children": [
    ]
}

Delete a notification

Delete a notification

Authorizations:
basicAuth
path Parameters
notification_id
required
string

Responses

Response samples

Content type
application/json
{
  • "error": {
    }
}
================================================ FILE: src/backend/interfaces/NATS/go.mockednats/nats.go ================================================ package mockednats import ( "errors" "github.com/nats-io/gnatsd/server" "github.com/nats-io/go-nats" "github.com/phayes/freeport" "strconv" "time" ) const ( natsUrl = "0.0.0.0" ) // GetNats starts an embedded nats server on localhost, // picking a free available port. func GetNats() (*server.Server, *nats.Conn, error) { // starting an embedded nats server port, err := freeport.GetFreePort() if err != nil { return nil, nil, err } natsServer, err := server.NewServer(&server.Options{ Host: natsUrl, Port: port, HTTPPort: -1, Cluster: server.ClusterOpts{Port: -1}, NoLog: true, NoSigs: true, Debug: false, Trace: false, }) if err != nil { return nil, nil, err } if natsServer == nil { return nil, nil, errors.New("natsServer is nil") } go natsServer.Start() // Wait for accept loop(s) to be started if !natsServer.ReadyForConnections(10 * time.Second) { return nil, nil, errors.New("timeout waiting nats server ready") } conn, err := nats.Connect("nats://" + natsUrl + ":" + strconv.Itoa(port)) if err != nil { natsServer.Shutdown() return nil, nil, err } return natsServer, conn, nil } ================================================ FILE: src/backend/interfaces/NATS/py.client/CHANGES.rst ================================================ 0.0.1 ----- - initial version ================================================ FILE: src/backend/interfaces/NATS/py.client/MANIFEST.in ================================================ include *.txt *.ini *.cfg *.rst ================================================ FILE: src/backend/interfaces/NATS/py.client/README.rst ================================================ Entry point =========== This repository is part of CaliOpen platform. For documentation, installation and contribution instructions, please refer to https://caliopen.github.io nats client ============ caliopen_nats package is an interface for py.main to subscribe to "topics" published on NATS server. For now, it listens for messages with 'inboundSMTP' subject. It triggers the inbound emails processing. to launch the daemon : ``` $ cd caliopen_nats/ $ python listener.py -f ../../../../configs/caliopen.yaml.template ``` ================================================ FILE: src/backend/interfaces/NATS/py.client/caliopen_nats/__init__.py ================================================ # -*- coding: utf-8 -*- __version__ = '0.23.0' ================================================ FILE: src/backend/interfaces/NATS/py.client/caliopen_nats/delivery.py ================================================ # -*- coding: utf-8 -*- """Caliopen user message delivery logic.""" from __future__ import absolute_import, print_function, unicode_literals import logging import uuid import datetime import pytz from caliopen_storage.exception import NotFound, DuplicateObject from caliopen_main.message.core import RawMessage from caliopen_main.message.core import MessageExternalRefLookup as Merl from caliopen_main.message.objects.message import Message from caliopen_main.user.store.tag import UserTag as Tag from caliopen_pi.qualifiers import UserMessageQualifier, UserTwitterQualifier, UserMastodonQualifier log = logging.getLogger(__name__) DUPLICATE_MESSAGE_EXC = "message already imported for this user" class UserMessageDelivery(object): def __init__(self, user, identity): """Create a new UserMessageDelivery belong to an user.""" self.user = user self.identity = identity self.qualifier = None def process_raw(self, raw_msg_id): """Process a raw message for an user, ie makes it a rich 'message'.""" raw = RawMessage.get(raw_msg_id) if not raw: log.error('Raw message <{}> not found'.format(raw_msg_id)) raise NotFound log.debug('Retrieved raw message {}'.format(raw_msg_id)) message = self.qualifier.process_inbound(raw) # fill user_tag table with imported tags and embed in message for tag in message.ext_tags: try: Tag.get(user_id=self.user.user_id, name=tag.name) except NotFound: try: Tag.create(user_id=self.user.user_id, name=tag.name, label=tag.label, type=tag.type, date_insert=datetime.datetime.now(tz=pytz.utc)) except Exception as exc: log.exception( "UserMessageDelivery failed to create tag : {}".format( exc)) message.tags.append(tag.name) if message.external_msg_id: external_refs = Merl._model_class.filter( user_id=self.user.user_id, external_msg_id=message.external_msg_id) if external_refs: msg = external_refs[0] # message already imported, should update it with identity_id ? obj = Message(user=self.user, message_id=msg.message_id) if str(msg.identity_id) != self.identity.identity_id: obj.get_db() obj.unmarshall_db() obj.user_identities.append(self.identity.identity_id) obj.marshall_db() obj.save_db() obj.marshall_index() obj.save_index() Merl.create(self.user, external_msg_id=msg.external_msg_id, identity_id=self.identity.identity_id, message_id=msg.message_id) # TODO: update flags ? raise DuplicateObject(DUPLICATE_MESSAGE_EXC) else: log.warn('Message without external message_id for raw {}'. format(raw.raw_msg_id)) # store and index Message obj = Message(user=self.user) obj.unmarshall_dict(message.to_native()) obj.user_id = uuid.UUID(self.user.user_id) obj.user_identities = [uuid.UUID(self.identity.identity_id)] obj.message_id = uuid.uuid4() obj.date_insert = datetime.datetime.now(tz=pytz.utc) obj.date_sort = obj.date_insert obj.marshall_db() obj.save_db() obj.marshall_index() obj.save_index() if message.external_msg_id: # store external_msg_id in lookup table # but do not abort if it failed try: Merl.create(self.user, external_msg_id=obj.external_msg_id, identity_id=obj.user_identity, message_id=obj.message_id) except Exception as exc: log.exception("UserMessageDelivery failed " "to store message_external_ref : {}".format(exc)) return obj class UserMailDelivery(UserMessageDelivery): """User email delivery processing.""" def __init__(self, user, identity): super(UserMailDelivery, self).__init__(user, identity) self.qualifier = UserMessageQualifier(self.user, self.identity) class UserTwitterDelivery(UserMessageDelivery): """Twitter Direct Message delivery processing""" def __init__(self, user, identity): super(UserTwitterDelivery, self).__init__(user, identity) self.qualifier = UserTwitterQualifier(self.user, self.identity) class UserMastodonDelivery(UserMessageDelivery): """Mastodon Direct Message delivery processing""" def __init__(self, user, identity): super(UserMastodonDelivery, self).__init__(user, identity) self.qualifier = UserMastodonQualifier(self.user, self.identity) ================================================ FILE: src/backend/interfaces/NATS/py.client/caliopen_nats/listener.py ================================================ # -*- coding: utf-8 -*- """Caliopen NATS listener for message processing.""" from __future__ import absolute_import, print_function, unicode_literals import argparse import sys import logging import tornado.ioloop import tornado.gen from nats.io import Client as Nats from caliopen_storage.config import Configuration from caliopen_storage.helpers.connection import connect_storage log = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) @tornado.gen.coroutine def inbound_smtp_handler(config): """Inbound message NATS handler.""" client = Nats() server = 'nats://{}:{}'.format(config['host'], config['port']) servers = [server] opts = {"servers": servers} yield client.connect(**opts) # create and register subscriber(s) inbound_email_sub = subscribers.InboundEmail(client) future = client.subscribe("inboundSMTP", "SMTPqueue", inbound_email_sub.handler) log.info("nats subscription started for inboundSMTP") future.result() @tornado.gen.coroutine def inbound_twitter_handler(config): """Inbound twitterDM NATS handler""" client = Nats() server = 'nats://{}:{}'.format(config['host'], config['port']) servers = [server] opts = {"servers": servers} yield client.connect(**opts) # create and register subscriber(s) inbound_twitter_sub = subscribers.InboundTwitter(client) future = client.subscribe("inboundTwitter", "Twitterqueue", inbound_twitter_sub.handler) log.info("nats subscription started for inboundTwitter") future.result() @tornado.gen.coroutine def inbound_mastodon_handler(config): """Inbound mastodonDM NATS handler""" client = Nats() server = 'nats://{}:{}'.format(config['host'], config['port']) servers = [server] opts = {"servers": servers} yield client.connect(**opts) # create and register subscriber(s) inbound_mastodon_sub = subscribers.InboundMastodon(client) future = client.subscribe("inboundMastodon", "Mastodonqueue", inbound_mastodon_sub.handler) log.info("nats subscription started for inboundMastodon") future.result() @tornado.gen.coroutine def contact_handler(config): """NATS handler for contact update events.""" client = Nats() server = 'nats://{}:{}'.format(config['host'], config['port']) servers = [server] opts = {"servers": servers} yield client.connect(**opts) # create and register subscriber(s) contact_subscriber = subscribers.ContactAction(client) future = client.subscribe("contactAction", "contactQueue", contact_subscriber.handler) log.info("nats subscription started for contactAction") future.result() @tornado.gen.coroutine def key_handler(config): """NATS handler for discover_key events.""" client = Nats() server = 'nats://{}:{}'.format(config['host'], config['port']) servers = [server] opts = {"servers": servers} yield client.connect(**opts) # create and register subscriber(s) key_subscriber = subscribers.KeyAction(client) future = client.subscribe("keyAction", "keyQueue", key_subscriber.handler) log.info("nats subscription started for keyAction") future.result() if __name__ == '__main__': # load Caliopen config args = sys.argv parser = argparse.ArgumentParser() parser.add_argument('-f', dest='conffile') kwargs = parser.parse_args(args[1:]) kwargs = vars(kwargs) Configuration.load(kwargs['conffile'], 'global') # need to load config before importing subscribers import subscribers connect_storage() inbound_smtp_handler(Configuration('global').get('message_queue')) inbound_twitter_handler(Configuration('global').get('message_queue')) inbound_mastodon_handler(Configuration('global').get('message_queue')) contact_handler(Configuration('global').get('message_queue')) key_handler(Configuration('global').get('message_queue')) loop_instance = tornado.ioloop.IOLoop.instance() loop_instance.start() ================================================ FILE: src/backend/interfaces/NATS/py.client/caliopen_nats/subscribers.py ================================================ # -*- coding: utf-8 -*- """Caliopen inbound nats message handler.""" from __future__ import absolute_import, print_function, unicode_literals import logging import json import sys import traceback from caliopen_storage.exception import DuplicateObject from caliopen_main.user.core import User, UserIdentity from caliopen_main.contact.objects import Contact from caliopen_nats.delivery import UserMailDelivery, UserTwitterDelivery, \ UserMastodonDelivery from caliopen_pi.qualifiers import ContactMessageQualifier from caliopen_pgp.keys import ContactPublicKeyManager from caliopen_main.common.core import PublicKey log = logging.getLogger(__name__) class BaseHandler(object): """Base class for NATS message handlers.""" def __init__(self, nats_cnx): """Create a new inbound messsage handler from a nats connection.""" self.natsConn = nats_cnx class InboundEmail(BaseHandler): """Inbound message class handler.""" def process_raw(self, msg, payload): """Process an inbound raw message.""" nats_error = { 'error': '', 'message': 'inbound email message process failed' } nats_success = { 'message': 'OK : inbound email message proceeded' } try: user = User.get(payload['user_id']) identity = UserIdentity.get(user, payload['identity_id']) deliver = UserMailDelivery(user, identity) new_message = deliver.process_raw(payload['message_id']) nats_success['message_id'] = str(new_message.message_id) nats_success['discussion_id'] = str(new_message.discussion_id) self.natsConn.publish(msg.reply, json.dumps(nats_success)) except DuplicateObject: log.info("Message already imported : {}".format(payload)) nats_success['message_id'] = str(payload['message_id']) nats_success['discussion_id'] = "" # message has not been parsed nats_success['message'] = 'raw message already imported' self.natsConn.publish(msg.reply, json.dumps(nats_success)) except Exception as exc: # TODO: handle abort exception and report it as special case exc_info = sys.exc_info() log.error("deliver process failed for raw {}: {}". format(payload, traceback.print_exception(*exc_info))) nats_error['error'] = str(exc) self.natsConn.publish(msg.reply, json.dumps(nats_error)) return exc def handler(self, msg): """Handle an process_raw nats messages.""" payload = json.loads(msg.data) log.info('Get payload order {}'.format(payload)) if payload['order'] == "process_raw": self.process_raw(msg, payload) else: log.warn( 'Unhandled payload order "{}" \ (queue: SMTPqueue, subject : inboundSMTP)'.format( payload['order'])) raise NotImplementedError class InboundTwitter(BaseHandler): """Inbound TwitterDM class handler.""" def process_raw(self, msg, payload): """Process an inbound raw message.""" nats_error = { 'error': '', 'message': 'inbound twitter message process failed' } nats_success = { 'message': 'OK : inbound twitter message proceeded' } try: user = User.get(payload['user_id']) identity = UserIdentity.get(user, payload['identity_id']) deliver = UserTwitterDelivery(user, identity) new_message = deliver.process_raw(payload['message_id']) nats_success['message_id'] = str(new_message.message_id) nats_success['discussion_id'] = str(new_message.discussion_id) self.natsConn.publish(msg.reply, json.dumps(nats_success)) except DuplicateObject: log.info("Message already imported : {}".format(payload)) nats_success['message_id'] = str(payload['message_id']) nats_success['discussion_id'] = "" # message has not been parsed nats_success['message'] = 'raw message already imported' self.natsConn.publish(msg.reply, json.dumps(nats_success)) except Exception as exc: # TODO: handle abort exception and report it as special case exc_info = sys.exc_info() log.error("deliver process failed for raw {}: {}". format(payload, traceback.print_exception(*exc_info))) nats_error['error'] = str(exc.message) self.natsConn.publish(msg.reply, json.dumps(nats_error)) return exc def handler(self, msg): """Handle an process_raw nats messages.""" payload = json.loads(msg.data) log.info('Get payload order {}'.format(payload['order'])) if payload['order'] == "process_raw": self.process_raw(msg, payload) else: log.warn('Unhandled payload type {}'.format(payload['order'])) class InboundMastodon(BaseHandler): """Inbound MastodonDM class handler.""" def process_raw(self, msg, payload): """Process an inbound raw message.""" nats_error = { 'error': '', 'message': 'inbound mastodon message process failed' } nats_success = { 'message': 'OK : inbound mastodon message proceeded' } try: user = User.get(payload['user_id']) identity = UserIdentity.get(user, payload['identity_id']) deliver = UserMastodonDelivery(user, identity) new_message = deliver.process_raw(payload['message_id']) nats_success['message_id'] = str(new_message.message_id) nats_success['discussion_id'] = str(new_message.discussion_id) self.natsConn.publish(msg.reply, json.dumps(nats_success)) except DuplicateObject: log.info("Message already imported : {}".format(payload)) nats_success['message_id'] = str(payload['message_id']) nats_success['discussion_id'] = "" # message has not been parsed nats_success['message'] = 'raw message already imported' self.natsConn.publish(msg.reply, json.dumps(nats_success)) except Exception as exc: # TODO: handle abort exception and report it as special case exc_info = sys.exc_info() log.error("deliver process failed for raw {}: {}". format(payload, traceback.print_exception(*exc_info))) nats_error['error'] = str(exc.message) self.natsConn.publish(msg.reply, json.dumps(nats_error)) return exc def handler(self, msg): """Handle an process_raw nats messages.""" payload = json.loads(msg.data) log.info('Get payload order {}'.format(payload['order'])) if payload['order'] == "process_raw": self.process_raw(msg, payload) else: log.warn('Unhandled payload type {}'.format(payload['order'])) class ContactAction(BaseHandler): """Handler for contact action message.""" def process_update(self, msg, payload): """Process a contact update message.""" # XXX validate payload structure if 'user_id' not in payload or 'contact_id' not in payload: raise Exception('Invalid contact_update structure') user = User.get(payload['user_id']) contact = Contact(user, contact_id=payload['contact_id']) contact.get_db() contact.unmarshall_db() qualifier = ContactMessageQualifier(user) log.info('Will process update for contact {0} of user {1}'. format(contact.contact_id, user.user_id)) # TODO: (re)discover GPG keys qualifier.process(contact) def handler(self, msg): """Handle an process_raw nats messages.""" payload = json.loads(msg.data) # log.info('Get payload order {}'.format(payload['order'])) if payload['order'] == "contact_update": self.process_update(msg, payload) else: log.warn( 'Unhandled payload order "{}" \ (queue: contactQueue, subject : contactAction)'.format( payload['order'])) raise NotImplementedError class KeyAction(BaseHandler): """Handler for public key discovery message.""" def _process_key(self, user, contact, key): if not key.is_expired: if key.userids: label = key.userids[0].name else: label = '{0} {1}'.format(key.algorithm, key.size) pub = PublicKey.create(user, contact.contact_id, 'contact', fingerprint=key.fingerprint, key=key.armored_key, expire_date=key.expire_date, label=label) log.info('Created public key {0}'.format(pub.key_id)) def _process_results(self, user, contact, results): fingerprints = [] for result in results: for key in result.keys: log.debug('Processing key %r' % key) if key.fingerprint not in fingerprints: self._process_key(user, contact, key) fingerprints.append(key.fingerprint) def process_key_discovery(self, msg, payload): """Discover public keys related to a new contact identifier.""" if 'user_id' not in payload or 'contact_id' not in payload: raise Exception('Invalid contact_update structure') user = User.get(payload['user_id']) contact = Contact(user.user_id, contact_id=payload['contact_id']) contact.get_db() contact.unmarshall_db() manager = ContactPublicKeyManager() founds = [] for ident in payload.get('emails', []): log.info('Process email identity {0}'.format(ident['address'])) discovery = manager.process_identity(user, contact, ident['address'], 'email') if discovery: founds.append(discovery) for ident in payload.get('identities', []): log.info('Process identity {0}:{1}'. format(ident['type'], ident['name'])) discovery = manager.process_identity(user, contact, ident['name'], ident['type']) if discovery: founds.append(discovery) if founds: log.info('Found %d results' % len(founds)) self._process_results(user, contact, founds) def handler(self, msg): """Handle a discover_key nats messages.""" payload = json.loads(msg.data) if payload['order'] == "discover_key": self.process_key_discovery(msg, payload) else: log.warn( 'Unhandled payload order "{}" \ (queue : keyQueue, subject : keyAction)'.format( payload['order'])) raise NotImplementedError ================================================ FILE: src/backend/interfaces/NATS/py.client/requirements.deps ================================================ caliopen_main ================================================ FILE: src/backend/interfaces/NATS/py.client/setup.cfg ================================================ [nosetests] match = ^test nocapture = 1 cover-package = caliop with-coverage = 1 cover-erase = 1 [compile_catalog] directory = caliop/locale domain = caliop statistics = true [extract_messages] add_comments = TRANSLATORS: output_file = caliop/locale/caliop.pot width = 80 [init_catalog] domain = caliop input_file = caliop/locale/caliop.pot output_dir = caliop/locale [update_catalog] domain = caliop input_file = caliop/locale/caliop.pot output_dir = caliop/locale previous = true ================================================ FILE: src/backend/interfaces/NATS/py.client/setup.py ================================================ import os import sys import re from setuptools import setup, find_packages PY3 = sys.version_info[0] == 3 name = "caliopen_nats" here = os.path.abspath(os.path.dirname(__file__)) README = open(os.path.join(here, 'README.rst')).read() CHANGES = open(os.path.join(here, 'CHANGES.rst')).read() with open(os.path.join(*([here] + name.split('.') + ['__init__.py']))) as v_file: version = re.compile(r".*__version__ = '(.*?)'", re.S).match(v_file.read()).group(1) requires = [ 'nats-client>=0.8.4', 'tornado==4.2',] if (os.path.isfile('./requirements.deps')): with open('./requirements.deps') as f_deps: requires.extend(f_deps.read().split('\n')) tests_require = [] if sys.version_info < (3, 3): tests_require.append('mock') extras_require = { 'dev': [], 'doc': [], 'test': tests_require } setup(name=name, version=version, description='subscription service to NATS topics', long_description=README + '\n\n' + CHANGES, classifiers=[ "Programming Language :: Python", "Topic :: Internet :: WWW/HTTP :: NATS", "Topic :: Internet :: WWW/HTTP :: WSGI :: Application", ], author='Caliopen Contributors', author_email='', url='https://github.com/CaliOpen/Caliopen/src/backend/interface/NATS/py.client', license='AGPLv3', keywords='nats', packages=find_packages(), include_package_data=True, zip_safe=False, install_requires=requires, tests_require=tests_require, extras_require=extras_require, test_suite="", entry_points={ 'paste.app_factory': ['main = nats_client:listener'], }) ================================================ FILE: src/backend/interfaces/REST/go.server/README.md ================================================ #### HTTP REST front-end interface for Caliopen application This package launches a HTTP server to interact with Caliopen services. By default, a proxy server runs in front of the http server to route requests to py.server or go.server as needed. For now, routes /api/v1/ are handled by py.server and routes /api/v2 by go server. ##### Installation A list of required dependencies is in `vendor/vendor.json`. To install go dependencies, run `govendor sync` from within `src/backend/interfaces/REST/go.server` directory. To compile the binary run `go build github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/cmd/caliopen_rest` ##### Usage Configuration files are in `src/backend/configs`. To launch the server from source files : ``` $ cd cmd/caliopen_rest/ $ go run main.go serve ``` To disable the proxy server : `go run main.go serve --proxy=false` ================================================ FILE: src/backend/interfaces/REST/go.server/api_server.go ================================================ // Copyleft (ɔ) 2018 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package rest_api import ( obj "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations/contacts" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations/devices" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations/discussions" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations/identities" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations/imports" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations/messages" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations/notifications" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations/participants" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations/providers" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations/tags" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations/users" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" log "github.com/Sirupsen/logrus" "github.com/gin-gonic/gin" "github.com/go-openapi/loads" "os" ) var ( server *REST_API ) type ( REST_API struct { config APIConfig swagSpec *loads.Document } APIConfig struct { Interface string `mapstructure:"listen_interface"` ListenPort string `mapstructure:"listen_port"` Port string `mapstructure:"port"` Hostname string `mapstructure:"hostname"` SwaggerFile string `mapstructure:"swaggerSpec"` BackendConfig `mapstructure:"BackendConfig"` IndexConfig `mapstructure:"IndexConfig"` CacheSettings `mapstructure:"RedisConfig"` NatsConfig `mapstructure:"NatsConfig"` NotifierConfig `mapstructure:"NotifierConfig"` Providers []obj.Provider `mapstructure:"Providers"` } BackendConfig struct { BackendName string `mapstructure:"backend_name"` Settings BackendSettings `mapstructure:"backend_settings"` } BackendSettings struct { Hosts []string `mapstructure:"hosts"` Keyspace string `mapstructure:"keyspace"` Consistency uint16 `mapstructure:"consistency_level"` SizeLimit uint64 `mapstructure:"raw_size_limit"` // max size for db (in bytes) ObjStoreType string `mapstructure:"object_store"` ObjStoreSettings obj.OSSConfig `mapstructure:"object_store_settings"` UseVault bool `mapstructure:"use_vault"` VaultSettings obj.VaultConfig `mapstructure:"vault_settings"` } IndexConfig struct { IndexName string `mapstructure:"index_name"` Settings IndexSettings `mapstructure:"index_settings"` } IndexSettings struct { Hosts []string `mapstructure:"hosts"` } CacheSettings struct { Host string `mapstructure:"host"` Password string `mapstructure:"password"` Db int `mapstructure:"db"` } NatsConfig struct { Url string `mapstructure:"url"` NatsQueue string `mapstructure:"nats_queue"` OutSMTP_topic string `mapstructure:"outSMTP_topic"` OutIMAP_topic string `mapstructure:"outIMAP_topic"` OutTWITTER_topic string `mapstructure:"outTWITTER_topic"` OutMASTODON_topic string `mapstructure:"outMASTODON_topic"` Contacts_topic string `mapstructure:"contacts_topic"` Keys_topic string `mapstructure:"keys_topic"` Users_topic string `mapstructure:"users_topic"` IdPoller_topic string `mapstructure:"idpoller_topic"` } NotifierConfig struct { AdminUsername string `mapstructure:"admin_username"` BaseUrl string `mapstructure:"base_url"` TemplatesPath string `mapstructure:"templates_path"` } ) func InitializeServer(config APIConfig) error { server = new(REST_API) return server.initialize(config) } func (server *REST_API) initialize(config APIConfig) error { server.config = config //init Caliopen facility caliopenConfig := obj.CaliopenConfig{ RESTstoreConfig: obj.RESTstoreConfig{ BackendName: config.BackendName, Hosts: config.BackendConfig.Settings.Hosts, Keyspace: config.BackendConfig.Settings.Keyspace, Consistency: config.BackendConfig.Settings.Consistency, SizeLimit: config.BackendConfig.Settings.SizeLimit, ObjStoreType: config.BackendConfig.Settings.ObjStoreType, OSSConfig: config.BackendConfig.Settings.ObjStoreSettings, UseVault: config.BackendConfig.Settings.UseVault, VaultConfig: config.BackendConfig.Settings.VaultSettings, }, RESTindexConfig: obj.RESTIndexConfig{ IndexName: config.IndexConfig.IndexName, Hosts: config.IndexConfig.Settings.Hosts, }, CacheConfig: obj.CacheConfig{ Host: config.CacheSettings.Host, Password: config.CacheSettings.Password, Db: config.CacheSettings.Db, }, NatsConfig: obj.NatsConfig{ Url: config.NatsConfig.Url, NatsQueue: config.NatsConfig.NatsQueue, OutSMTP_topic: config.NatsConfig.OutSMTP_topic, OutIMAP_topic: config.NatsConfig.OutIMAP_topic, OutTWITTER_topic: config.NatsConfig.OutTWITTER_topic, OutMASTODON_topic: config.NatsConfig.OutMASTODON_topic, Contacts_topic: config.NatsConfig.Contacts_topic, Keys_topic: config.NatsConfig.Keys_topic, Users_topic: config.NatsConfig.Users_topic, IdPoller_topic: config.NatsConfig.IdPoller_topic, }, NotifierConfig: obj.NotifierConfig{ AdminUsername: config.NotifierConfig.AdminUsername, BaseUrl: config.NotifierConfig.BaseUrl, TemplatesPath: config.NotifierConfig.TemplatesPath, }, Providers: config.Providers, Hostname: config.Hostname + ":" + config.Port, } err := caliopen.Initialize(caliopenConfig) if err != nil { log.WithError(err).Fatal("Caliopen facilities initialization failed") } //checks that with could open the swagger specs file _, err = os.Stat(server.config.SwaggerFile) if err != nil { return err } return nil } func StartServer() error { return server.start() } func (server *REST_API) start() error { // Creates a gin router with default middleware: // logger and recovery (crash-free) middleware router := gin.Default() //router.Use(Dumper()) // adds our middlewares err := http_middleware.InitSwaggerMiddleware(server.config.SwaggerFile) if err != nil { log.WithError(err).Warn("init swagger middleware failed") } else { router.Use(http_middleware.SwaggerValidator()) } // adds our routes and handlers api := router.Group(http_middleware.RoutePrefix) server.AddHandlers(api) // listens addr := server.config.Interface + ":" + server.config.ListenPort err = router.Run(addr) if err != nil { log.WithError(err).Warn("unable to start gin server") } return err } func (server *REST_API) AddHandlers(api *gin.RouterGroup) { /** users API **/ usrs := api.Group("/users", http_middleware.BasicAuthFromCache(caliopen.Facilities.Cache, "caliopen")) usrs.PATCH("/:user_id", users.PatchUser) usrs.POST("/:user_id/actions", users.Delete) /** identities **/ ids := api.Group(http_middleware.IdentitiesRoute, http_middleware.BasicAuthFromCache(caliopen.Facilities.Cache, "caliopen")) ids.GET("/locals", identities.GetLocalsIdentities) ids.GET("/locals/:identity_id", identities.GetLocalIdentity) ids.GET("/remotes", identities.GetRemoteIdentities) ids.POST("/remotes", identities.NewRemoteIdentity) ids.GET("/remotes/:remote_id", identities.GetRemoteIdentity) ids.PATCH("/remotes/:remote_id", identities.PatchRemoteIdentity) ids.DELETE("/remotes/:remote_id", identities.DeleteRemoteIdentity) /** passwords API **/ passwords := api.Group("/passwords") passwords.GET("/reset", notImplemented) passwords.POST("/reset", users.RequestPasswordReset) passwords.GET("/reset/:reset_token", users.ValidatePassResetToken) passwords.POST("/reset/:reset_token", users.ResetPassword) /** username API **/ api.GET("/username/isAvailable", users.IsAvailable) /** messages API **/ msg := api.Group("/messages", http_middleware.BasicAuthFromCache(caliopen.Facilities.Cache, "caliopen")) msg.GET("", messages.GetMessagesList) msg.GET("/:message_id", messages.GetMessage) msg.POST("/:message_id/actions", messages.Actions) //attachments msg.POST("/:message_id/attachments", messages.UploadAttachment) msg.DELETE("/:message_id/attachments/:attachment_id", messages.DeleteAttachment) msg.GET("/:message_id/attachments/:attachment_id", messages.DownloadAttachment) //tags msg.PATCH("/:message_id/tags", tags.PatchResourceWithTags) /** discussions API **/ disc := api.Group("/discussions", http_middleware.BasicAuthFromCache(caliopen.Facilities.Cache, "caliopen")) disc.GET("", discussions.GetDiscussionsList) disc.GET("/:discussionId", discussions.GetDiscussion) /** participants API **/ parts := api.Group("/participants", http_middleware.BasicAuthFromCache(caliopen.Facilities.Cache, "caliopen")) parts.GET("/suggest", participants.Suggest) parts.POST("/discussion", participants.HashUris) /** contacts API **/ cts := api.Group(http_middleware.ContactsRoute, http_middleware.BasicAuthFromCache(caliopen.Facilities.Cache, "caliopen")) cts.GET("", contacts.GetContactsList) cts.POST("", contacts.NewContact) cts.GET("/:contactID", contacts.GetContact) cts.PATCH("/:contactID", contacts.PatchContact) cts.DELETE("/:contactID", contacts.DeleteContact) cts.GET("/:contactID/identities", contacts.GetIdentities) //publickeys cts.POST("/:contactID/publickeys", contacts.NewPublicKey) cts.GET("/:contactID/publickeys", contacts.GetPubKeys) cts.GET("/:contactID/publickeys/:pubkeyID", contacts.GetPubKey) cts.PATCH("/:contactID/publickeys/:pubkeyID", contacts.PatchPubKey) cts.DELETE("/:contactID/publickeys/:pubkeyID", contacts.DeletePubKey) //tags cts.PATCH("/:contactID/tags", tags.PatchResourceWithTags) /** devices API **/ api.GET("/validate-device/:token", http_middleware.BasicAuthFromCache(caliopen.Facilities.Cache, "caliopen"), devices.ValidateDevice) dev := api.Group(http_middleware.DevicesRoute, http_middleware.BasicAuthFromCache(caliopen.Facilities.Cache, "caliopen")) dev.GET("", devices.GetDevicesList) //dev.POST("", devices.NewDevice) dev.GET("/:deviceID", devices.GetDevice) dev.PATCH("/:deviceID", devices.PatchDevice) dev.DELETE("/:deviceID", devices.DeleteDevice) dev.POST("/:deviceID/actions", devices.Actions) // imports imp := api.Group(http_middleware.ImportsRoute, http_middleware.BasicAuthFromCache(caliopen.Facilities.Cache, "caliopen")) imp.POST("", imports.ImportFile) /** tags API **/ tag := api.Group(http_middleware.TagsRoute, http_middleware.BasicAuthFromCache(caliopen.Facilities.Cache, "caliopen")) tag.GET("", tags.RetrieveUserTags) tag.POST("", tags.CreateTag) tag.GET("/:tag_name", tags.RetrieveTag) tag.PATCH("/:tag_name", tags.PatchTag) tag.DELETE("/:tag_name", tags.DeleteTag) /** search API **/ search := api.Group("/search", http_middleware.BasicAuthFromCache(caliopen.Facilities.Cache, "caliopen")) search.GET("", operations.SimpleSearch) search.POST("", operations.AdvancedSearch) /** notifications API **/ notif := api.Group("/notifications", http_middleware.BasicAuthFromCache(caliopen.Facilities.Cache, "caliopen")) notif.GET("", notifications.GetPendingNotif) notif.DELETE("", notifications.DeleteNotifications) notif.GET("/:notification_id", notifications.GetNotification) notif.DELETE("/:notification_id", notifications.DeleteNotification) /** providers **/ prov := api.Group("/providers") prov.GET("", http_middleware.BasicAuthFromCache(caliopen.Facilities.Cache, "caliopen"), providers.GetProvidersList) prov.GET("/:provider_name", http_middleware.BasicAuthFromCache(caliopen.Facilities.Cache, "caliopen"), providers.GetProvider) prov.GET("/:provider_name/callback", providers.CallbackHandler) api.StaticFile("/test/oauth", "../interfaces/REST/go.server/operations/providers/oauth-test.html") } ================================================ FILE: src/backend/interfaces/REST/go.server/cmd/caliopen_rest/cli_cmds/root.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package cmd import ( log "github.com/Sirupsen/logrus" "github.com/spf13/cobra" ) var ( verbose bool version bool RootCmd = &cobra.Command{ Use: "caliopen_rest", Short: "Caliopen REST HTTP API", Long: `HTTP server (& proxy) for the frontend applications`, Run: nil, } ) const __version__ = "0.26.1" func init() { cobra.OnInitialize() RootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "print out more debug information") RootCmd.PersistentFlags().BoolVarP(&version, "version", "V", false, "print out the version of this program") RootCmd.Run = func(cmd *cobra.Command, args []string) { if version { log.Infof("Caliopen APIv2 version %s", __version__) } } RootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { if verbose { log.SetLevel(log.DebugLevel) } else { log.SetLevel(log.InfoLevel) } } RootCmd.AddCommand(versionCmd) } var versionCmd = &cobra.Command{ Use: "version", Short: "Print the version number of Caliopen REST api", Long: `All software has versions. This is Caliopen API's`, Run: func(cmd *cobra.Command, args []string) { log.Infof("Caliopen REST API version %s", __version__) }, } ================================================ FILE: src/backend/interfaces/REST/go.server/cmd/caliopen_rest/cli_cmds/serve.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. // APIs() launches the Caliopen application's interfaces processes package cmd import ( "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server" log "github.com/Sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" "os" "os/signal" "syscall" ) var ( withProxy bool configFileName string configPath string pidFile string cmdConfig CmdConfig serveCmd = &cobra.Command{ Use: "serve", Short: "start the caliopen REST HTTP API server (& proxy)", Run: API, } signalChannel = make(chan os.Signal, 1) // for trapping SIG_HUP ) func init() { cobra.OnInitialize() serveCmd.PersistentFlags().StringVarP(&configFileName, "configfile", "c", "caliopen-go-api_dev", "Name of the configuration file, without extension. (YAML, TOML, JSON… allowed)") serveCmd.PersistentFlags().StringVarP(&configPath, "configpath", "", "../../../../../configs/", "API config file path.") serveCmd.PersistentFlags().StringVarP(&pidFile, "pid-file", "p", "/var/run/caliopen_rest.pid", "Path to the pid file") serveCmd.PersistentFlags().BoolVarP(&withProxy, "proxy", "", false, "Start HTTP proxy for routing to both GO & Python services") serveCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { if verbose { log.SetLevel(log.DebugLevel) } else { log.SetLevel(log.InfoLevel) } } RootCmd.AddCommand(serveCmd) signalChannel = make(chan os.Signal, 1) } func sigHandler() { // handle SIGHUP for reloading the configuration while running signal.Notify(signalChannel, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGINT, syscall.SIGKILL) for sig := range signalChannel { if sig == syscall.SIGHUP { err := readConfig(false) if err != nil { log.WithError(err).Error("Error while ReadConfig (reload)") } else { log.Infof("Configuration is reloaded") } // TODO: reinitialize } else if sig == syscall.SIGTERM || sig == syscall.SIGQUIT || sig == syscall.SIGINT { log.Info("Shutdown signal caught") if withProxy { } //app.Shutdown() log.Infof("Shutdown completed, exiting.") os.Exit(0) } else { os.Exit(0) } } } func API(cmd *cobra.Command, args []string) { err := readConfig(false) if withProxy { // start HTTP reverse proxy go rest_api.StartProxy(cmdConfig.ProxyConfig) } err = rest_api.InitializeServer(cmdConfig.APIConfig) if err != nil { log.Fatal(err) } go rest_api.StartServer() sigHandler() } // Read and parse api configuration file func readConfig(readAll bool) error { // load in the main config. Reading from YAML, TOML, JSON, HCL and Java properties config files apiViper := viper.New() apiViper.SetConfigName(configFileName) apiViper.AddConfigPath(configPath) apiViper.AddConfigPath("$CALIOPENROOT/src/backend/configs/") apiViper.AddConfigPath(".") // load APIs config err := apiViper.ReadInConfig() if err != nil { log.WithError(err).Infof("Could not read api config file <%s>.", configFileName) return err } err = apiViper.Unmarshal(&cmdConfig) if err != nil { log.WithError(err).Infof("Could not parse api config file: <%s>", configFileName) return err } return nil } type CmdConfig struct { rest_api.APIConfig rest_api.IndexConfig rest_api.ProxyConfig } ================================================ FILE: src/backend/interfaces/REST/go.server/cmd/caliopen_rest/main.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package main import ( "fmt" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/cmd/caliopen_rest/cli_cmds" "os" ) func main() { // launch 'root cmd' that will register other commands that could be executed if err := cmd.RootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(-1) } } ================================================ FILE: src/backend/interfaces/REST/go.server/dump_request.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package rest_api import ( log "github.com/Sirupsen/logrus" "github.com/gin-gonic/gin" ) func Dumper() gin.HandlerFunc { return func(c *gin.Context) { // before request log.Infof("Client IP : %s", c.ClientIP()) log.Infof("User Agent: %s", c.GetHeader("User-Agent")) c.Next() // after request } } ================================================ FILE: src/backend/interfaces/REST/go.server/errors.go ================================================ package rest_api import ( "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "net/http" ) type ( Errors []Error Error struct { // An error supports zero or more field names, because an // error can morph three ways: (1) it can indicate something // wrong with the request as a whole, (2) it can point to a // specific problem with a particular input field, or (3) it // can span multiple related input fields. FieldNames []string `json:"fieldNames,omitempty"` // The classification is like an error code, convenient to // use when processing or categorizing an error programmatically. // It may also be called the "kind" of error. Classification string `json:"classification,omitempty"` // Message should be human-readable and detailed enough to // pinpoint and resolve the problem, but it should be brief. For // example, a payload of 100 objects in a JSON array might have // an error in the 41st object. The message should help the // end user find and fix the error with their request. Message string `json:"message,omitempty"` } ) func notImplemented(ctx *gin.Context) { e := swgErr.New(http.StatusNotImplemented, "not implemented") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } ================================================ FILE: src/backend/interfaces/REST/go.server/middlewares/authentication.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package http_middleware import ( "crypto/ecdsa" "crypto/elliptic" "crypto/sha256" "encoding/asn1" "encoding/base64" "errors" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" log "github.com/Sirupsen/logrus" "github.com/gin-gonic/gin" "math/big" "net/http" "strconv" "strings" "time" ) type ecdsaSignature struct { R, S *big.Int } // getSignedQuery, build the HTTP query that has been signed func getSignedQuery(c *gin.Context) string { query := c.Request.Method + c.Request.URL.String() return query } // verifySignature, check for validity of device ecdsa signature func verifySignature(signature, query, curve string, x, y big.Int) (bool, error) { sign := &ecdsaSignature{} decoded, err := base64.StdEncoding.DecodeString(signature) if err != nil { return false, err } _, err = asn1.Unmarshal([]byte(decoded), sign) if err != nil { return false, err } var hashed []byte var key ecdsa.PublicKey switch curve { case "P-256": // Create hash of content hash := sha256.New() hash.Write([]byte(query)) hashed = hash.Sum(nil) crv := elliptic.P256() key = ecdsa.PublicKey{Curve: crv, X: &x, Y: &y} default: return false, errors.New("Invalid device curve") } valid := ecdsa.Verify(&key, hashed, sign.R, sign.S) return valid, nil } func BasicAuthFromCache(cache backends.APICache, realm string) gin.HandlerFunc { if realm == "" { realm = "Authorization Required" } realm = "Basic realm=" + strconv.Quote(realm) return func(c *gin.Context) { // Get provided auth headers var user_id, access_token string var ok bool var cache_key string //Try auth-scheme 'Bearer' then 'Basic' if user_id, access_token, ok = BearerAuth(c.Request); !ok { if user_id, access_token, ok = c.Request.BasicAuth(); !ok { kickUnauthorizedRequest(c, realm) return } } if device_id := c.Request.Header.Get("X-Caliopen-Device-ID"); device_id != "" { cache_key = user_id + "-" + device_id } // Search user in cache of allowed credentials auth, err := cache.GetAuthToken("tokens::" + cache_key) if err != nil || auth == nil || auth.Access_token != access_token || time.Since(auth.Expires_at) > 0 { kickUnauthorizedRequest(c, realm) return } if auth.User_status == "maintenance" || auth.User_status == "locked" { c.AbortWithError(401, errors.New("User status does not permit operations")) } if device_sign := c.Request.Header.Get("X-Caliopen-Device-Signature"); device_sign != "" { query := getSignedQuery(c) valid, err := verifySignature(device_sign, query, auth.Curve, auth.X, auth.Y) if err != nil { log.Println("Error during signature verification: ", err) // kickUnauthorizedRequest(c, "Authorization error") } if valid == false { log.Println("Verification of signature failed") // kickUnauthorizedRequest(c, "Authorization failed") } } else { log.Println("No signature found for device ") } //save user_id in context for future retreival c.Set("user_id", user_id) c.Set("access_token", "tokens::"+cache_key) c.Set("shard_id", auth.Shard_id) } } func kickUnauthorizedRequest(c *gin.Context, realm string) { c.AbortWithStatus(401) } func BearerAuth(r *http.Request) (username, password string, ok bool) { auth := r.Header.Get("Authorization") if auth == "" { return } return parseBearerAuth(auth) } func parseBearerAuth(auth string) (username, password string, ok bool) { const prefix = "Bearer " if !strings.HasPrefix(auth, prefix) { return } c, err := base64.StdEncoding.DecodeString(auth[len(prefix):]) if err != nil { return } cs := string(c) s := strings.IndexByte(cs, ':') if s < 0 { return } return cs[:s], cs[s+1:], true } ================================================ FILE: src/backend/interfaces/REST/go.server/middlewares/config.go ================================================ package http_middleware const ( RoutePrefix = "/api/v2" IdentitiesRoute = "/identities" TagsRoute = "/tags" ContactsRoute = "/contacts" DevicesRoute = "/devices" ImportsRoute = "/imports" ) ================================================ FILE: src/backend/interfaces/REST/go.server/middlewares/swagger.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package http_middleware import ( "bytes" "encoding/json" "errors" log "github.com/Sirupsen/logrus" "github.com/gin-gonic/gin" "github.com/go-openapi/analysis" swgErr "github.com/go-openapi/errors" "github.com/go-openapi/loads" "github.com/go-openapi/runtime" "github.com/go-openapi/runtime/middleware" "github.com/go-openapi/runtime/middleware/untyped" "github.com/go-openapi/spec" "github.com/go-openapi/strfmt" "io" "io/ioutil" "net/http" "strings" "sync" ) var ( swaggerSpec *loads.Document swaggerAPI *routableUntypedAPI swaggerContext *middleware.Context ) const ( noWritten = -1 defaultStatus = 200 ) type routableUntypedAPI struct { api *untyped.API hlock *sync.Mutex handlers map[string]map[string]http.Handler defaultConsumes string defaultProduces string } type jsonError struct { Code int32 `json:"code"` Message string `json:"message"` Name string `json:"name"` } type jsonErrors []jsonError func InitSwaggerMiddleware(swaggerFile string) (err error) { log.Infoln("Loading swagger specifications…") swaggerSpec, err = loads.JSONSpec(swaggerFile) if err != nil { return err } swagAPI := untyped.NewAPI(swaggerSpec) swagAPI.WithJSONDefaults() swagAPI.RegisterConsumer("multipart/form-data", runtime.TextConsumer()) //TODO: write our consumers to be less tighted to open-api //swagAPI.ServeError = ServeError swagCtx := middleware.NewContext(swaggerSpec, swagAPI, nil) if swagCtx == nil { log.Warn("no swagContext") } swaggerAPI = newRoutableUntypedAPI(swaggerSpec, swagAPI, swagCtx) swagRouter := middleware.DefaultRouter(swaggerSpec, swaggerAPI) if swagRouter == nil { log.Warn("no swagRouter") } swaggerContext = middleware.NewRoutableContext(swaggerSpec, swaggerAPI, swagRouter) if swaggerContext == nil { log.Warn("no swagContext") } middleware.NewRouter(swaggerContext, nil) //workaround to set the router within swaggerContext return } // checks that inputs and/or outputs conform to the swagger specs for the route // this middleware should be registred as the first middleware to ensure that it checks // requests before next handlers func SwaggerValidator() gin.HandlerFunc { return func(ctx *gin.Context) { SwaggerInboundValidation(ctx) ctx.Next() //SwaggerOutboundValidation(ctx) } } func SwaggerInboundValidation(ctx *gin.Context) { // make a copy of request to be able to drain body twice : // one for swagger validation, other to ctx.next handlers body1, body2, err := drainBody(ctx.Request.Body) req_copy := new(http.Request) *req_copy = *ctx.Request ctx.Request.Body = body1 req_copy.Body = body2 if err != nil { ctx.Abort() return } route, ok := swaggerContext.RouteInfo(ctx.Request) if route != nil && ok { _, err := swaggerContext.BindAndValidate(req_copy, route) if err != nil { ServeError(ctx.Writer, ctx.Request, err) ctx.Abort() return } } else { ServeError(ctx.Writer, ctx.Request, errors.New("Route <"+ctx.Request.Method+" "+ctx.Request.RequestURI+"> not found in swagger specs.")) ctx.Abort() return } ctx.Set("swgCtx", swaggerContext) } func newRoutableUntypedAPI(spec *loads.Document, api *untyped.API, context *middleware.Context) *routableUntypedAPI { var handlers map[string]map[string]http.Handler if spec == nil || api == nil { return nil } analyzer := analysis.New(spec.Spec()) for method, hls := range analyzer.Operations() { um := strings.ToUpper(method) for path, op := range hls { schemes := analyzer.SecurityDefinitionsFor(op) if handlers == nil { handlers = make(map[string]map[string]http.Handler) } if b, ok := handlers[um]; !ok || b == nil { handlers[um] = make(map[string]http.Handler) } var handler http.Handler //fake handler as we won't use it if len(schemes) > 0 { handler = newSecureAPI(context, nil) } handlers[um][path] = handler } } return &routableUntypedAPI{ api: api, hlock: new(sync.Mutex), handlers: handlers, defaultProduces: api.DefaultProduces, defaultConsumes: api.DefaultConsumes, } } // ServeError the error handler interface implementation // returns an error json as defined within swagger.json, if any func ServeError(rw gin.ResponseWriter, r *http.Request, err error) { rw.Header().Set("Content-Type", "application/json") switch e := err.(type) { case *swgErr.CompositeError: er := flattenComposite(e) var lastCode int //get the last error code to return it to client if lastErr, ok := er.Errors[0].(swgErr.Error); ok { lastCode = int(lastErr.Code()) } else { lastCode = int(e.Code()) } rw.WriteHeader(asHTTPCode(lastCode)) if r == nil || r.Method != "HEAD" { rw.Write(errorAsJSON(er)) } case *swgErr.MethodNotAllowedError: rw.Header().Add("Allow", strings.Join(err.(*swgErr.MethodNotAllowedError).Allowed, ",")) rw.WriteHeader(asHTTPCode(int(e.Code()))) if r == nil || r.Method != "HEAD" { rw.Write(errorAsJSON(e)) } case swgErr.Error: if e == nil { rw.WriteHeader(http.StatusInternalServerError) rw.Write(errorAsJSON(swgErr.New(http.StatusInternalServerError, "Unknown error"))) return } rw.WriteHeader(asHTTPCode(int(e.Code()))) if r == nil || r.Method != "HEAD" { rw.Write(errorAsJSON(e)) } default: rw.WriteHeader(http.StatusInternalServerError) if r == nil || r.Method != "HEAD" { rw.Write(errorAsJSON(swgErr.New(http.StatusInternalServerError, err.Error()))) } } log.WithError(err).Error("Processing error") rw.Flush() } func errorAsJSON(err swgErr.Error) []byte { errs := struct { Errors jsonErrors `json:"errors"` }{} switch er := err.(type) { case *swgErr.CompositeError: for _, e := range er.Errors { if swgerr, ok := e.(swgErr.Error); ok { errs.Errors = append(errs.Errors, jsonError{swgerr.Code(), e.Error(), ""}) } else { errs.Errors = append(errs.Errors, jsonError{err.Code(), e.Error(), ""}) } } b, _ := json.Marshal(errs) return b default: errs.Errors = append(errs.Errors, jsonError{err.Code(), err.Error(), ""}) b, _ := json.Marshal(errs) return b } } func flattenComposite(errs *swgErr.CompositeError) *swgErr.CompositeError { var res []error for _, er := range errs.Errors { switch e := er.(type) { case *swgErr.CompositeError: if len(e.Errors) > 0 { flat := flattenComposite(e) if len(flat.Errors) > 0 { res = append(res, flat.Errors...) } } default: if e != nil { res = append(res, e) } } } return swgErr.CompositeValidationError(res...) } func asHTTPCode(input int) int { if input < 400 || input >= 600 { return 422 } return input } // drainBody reads all of b to memory and then returns two equivalent // ReadClosers yielding the same bytes. // // It returns an error if the initial slurp of all bytes fails. It does not attempt // to make the returned ReadClosers have identical error-matching behavior. func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) { if b == http.NoBody { // No copying needed. Preserve the magic sentinel meaning of NoBody. return http.NoBody, http.NoBody, nil } var buf bytes.Buffer if _, err = buf.ReadFrom(b); err != nil { return nil, b, err } if err = b.Close(); err != nil { return nil, b, err } return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil } // Func copied from go-openapi func newSecureAPI(ctx *middleware.Context, next http.Handler) http.Handler { return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { route, _ := ctx.RouteInfo(r) if route != nil && len(route.Authenticators) == 0 { next.ServeHTTP(rw, r) return } if _, err := ctx.Authorize(r, route); err != nil { ctx.Respond(rw, r, route.Produces, route, err) return } next.ServeHTTP(rw, r) }) } // Funcs below are copied from openapi sources to facilitate the swagger validation. // They allow us to satisfy the openapi "RoutableAPI" interface. // Our current implementation do not call them directly at anytime. // They should be removed in future. func (r *routableUntypedAPI) HandlerFor(method, path string) (http.Handler, bool) { r.hlock.Lock() paths, ok := r.handlers[strings.ToUpper(method)] if !ok { r.hlock.Unlock() return nil, false } handler, ok := paths[path] r.hlock.Unlock() return handler, ok } func (r *routableUntypedAPI) ServeErrorFor(operationID string) func(http.ResponseWriter, *http.Request, error) { return r.api.ServeError } func (r *routableUntypedAPI) ConsumersFor(mediaTypes []string) map[string]runtime.Consumer { return r.api.ConsumersFor(mediaTypes) } func (r *routableUntypedAPI) ProducersFor(mediaTypes []string) map[string]runtime.Producer { return r.api.ProducersFor(mediaTypes) } func (r *routableUntypedAPI) AuthenticatorsFor(schemes map[string]spec.SecurityScheme) map[string]runtime.Authenticator { return r.api.AuthenticatorsFor(schemes) } func (r *routableUntypedAPI) Formats() strfmt.Registry { return r.api.Formats() } func (r *routableUntypedAPI) DefaultProduces() string { return r.defaultProduces } func (r *routableUntypedAPI) DefaultConsumes() string { return r.defaultConsumes } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/contacts/Identities.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package contacts import ( "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "net/http" ) //GET …/contacts/{contactID}/identities func GetIdentities(ctx *gin.Context) { user_id := ctx.MustGet("user_id").(string) contact_id, err := operations.NormalizeUUIDstring(ctx.Param("contactID")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } identities, err := caliopen.Facilities.RESTfacility.RetrieveContactIdentities(user_id, contact_id) if err != nil { e := swgErr.New(http.StatusInternalServerError, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } ret := struct { Total int `json:"total"` ContactIdentities []objects.ContactIdentity `json:"contact_identities"` }{len(identities), identities} ctx.JSON(http.StatusOK, ret) } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/contacts/contacts.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package contacts import ( "bytes" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "github.com/satori/go.uuid" "io/ioutil" "net/http" "strconv" "strings" ) // GetContactList handles GET /contacts func GetContactsList(ctx *gin.Context) { var limit, offset int var user_UUID UUID var list []*Contact var totalFound int64 var err error user_uuid_str := ctx.MustGet("user_id").(string) user_uuid, _ := uuid.FromString(user_uuid_str) shard_id := ctx.MustGet("shard_id").(string) user_UUID.UnmarshalBinary(user_uuid.Bytes()) query_values := ctx.Request.URL.Query() if uriFilter, ok := query_values["uri"]; ok { // lookup contact by uri is made into store list, totalFound, err = caliopen.Facilities.RESTfacility.LookupContactByUri(user_uuid_str, uriFilter[0]) if err != nil && err.Error() != "not found" { e := swgErr.New(http.StatusInternalServerError, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } } else { if l, ok := query_values["limit"]; ok { limit, _ = strconv.Atoi(l[0]) query_values.Del("limit") } if o, ok := query_values["offset"]; ok { offset, _ = strconv.Atoi(o[0]) query_values.Del("offset") } filter := IndexSearch{ User_id: user_UUID, Shard_id: shard_id, Terms: map[string][]string(query_values), Limit: limit, Offset: offset, } list, totalFound, err = caliopen.Facilities.RESTfacility.RetrieveContacts(filter) if err != nil { e := swgErr.New(http.StatusInternalServerError, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } } // render response var respBuf bytes.Buffer respBuf.WriteString("{\"total\": " + strconv.FormatInt(totalFound, 10) + ",") respBuf.WriteString("\"contacts\":[") first := true for _, contact := range list { json_contact, err := contact.MarshalFrontEnd() if err == nil { if first { first = false } else { respBuf.WriteByte(',') } respBuf.Write(json_contact) } } respBuf.WriteString("]}") ctx.Data(http.StatusOK, "application/json; charset=utf-8", respBuf.Bytes()) } // NewContact handles POST /contacts func NewContact(ctx *gin.Context) { userId, err := operations.NormalizeUUIDstring(ctx.MustGet("user_id").(string)) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } shard_id := ctx.MustGet("shard_id").(string) user_info := &UserInfo{User_id: userId, Shard_id: shard_id} contact := new(Contact) contact.MarshallNew() err = ctx.ShouldBindJSON(contact) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } contact.UserId.UnmarshalBinary(uuid.FromStringOrNil(userId).Bytes()) err = caliopen.Facilities.RESTfacility.CreateContact(user_info, contact) if err != nil { var e error if strings.HasPrefix(err.Error(), "uri <") { e = swgErr.New(http.StatusForbidden, err.Error()) } else { e = swgErr.New(http.StatusInternalServerError, err.Error()) } http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.JSON(http.StatusOK, struct { Location string `json:"location"` ContactId string `json:"contact_id"` }{ http_middleware.RoutePrefix + http_middleware.ContactsRoute + "/" + contact.ContactId.String(), contact.ContactId.String(), }) } return } // GetContact handles GET /contacts/:contactID func GetContact(ctx *gin.Context) { userID := ctx.MustGet("user_id").(string) contactID, err := operations.NormalizeUUIDstring(ctx.Param("contactID")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } contact, err := caliopen.Facilities.RESTfacility.RetrieveContact(userID, contactID) if err != nil { e := swgErr.New(http.StatusInternalServerError, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } contact_json, err := contact.MarshalFrontEnd() if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Data(http.StatusOK, "application/json; charset=utf-8", contact_json) } } // PatchContact handles PATCH /contacts/:contactID func PatchContact(ctx *gin.Context) { var err error userId, err := operations.NormalizeUUIDstring(ctx.MustGet("user_id").(string)) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } shard_id := ctx.MustGet("shard_id").(string) user_info := &UserInfo{User_id: userId, Shard_id: shard_id} contactId, err := operations.NormalizeUUIDstring(ctx.Param("contactID")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } var patch []byte patch, err = ioutil.ReadAll(ctx.Request.Body) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // call REST facility with payload err = caliopen.Facilities.RESTfacility.PatchContact(user_info, patch, contactId) if err != nil { if Cerr, ok := err.(CaliopenError); ok { returnedErr := new(swgErr.CompositeError) if Cerr.Code() == FailDependencyCaliopenErr { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "[RESTfacility] PatchContact failed"), Cerr, Cerr.Cause()) } else if Cerr.Code() == ForbiddenCaliopenErr { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusForbidden, "[RESTfacility] PatchContact forbidden"), Cerr, Cerr.Cause()) } else { returnedErr = swgErr.CompositeValidationError(Cerr, Cerr.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } else { ctx.Status(http.StatusNoContent) } } // DeleteContact handles DELETE /contacts/:contactID func DeleteContact(ctx *gin.Context) { userId, err := operations.NormalizeUUIDstring(ctx.MustGet("user_id").(string)) contactID, err := operations.NormalizeUUIDstring(ctx.Param("contactID")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } shard_id := ctx.MustGet("shard_id").(string) user_info := &UserInfo{User_id: userId, Shard_id: shard_id} err = caliopen.Facilities.RESTfacility.DeleteContact(user_info, contactID) if err != nil { e := swgErr.New(http.StatusInternalServerError, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } ctx.Status(http.StatusNoContent) } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/contacts/keys.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package contacts import ( "bytes" "encoding/base64" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "github.com/satori/go.uuid" "io/ioutil" "net/http" "strconv" ) // NewPublicKey handles POST …/contacts/:contactID/publickeys func NewPublicKey(ctx *gin.Context) { // check payload userId := ctx.MustGet("user_id").(string) userId, err := operations.NormalizeUUIDstring(userId) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } contactId := ctx.Param("contactID") if contactId == "" { e := swgErr.New(http.StatusUnprocessableEntity, "empty contactID") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } contactId, err = operations.NormalizeUUIDstring(contactId) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } contact, err := caliopen.Facilities.RESTfacility.RetrieveContact(userId, contactId) if err != nil { e := swgErr.New(http.StatusNotFound, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } payload := struct { Label string Key string }{} err = ctx.ShouldBindJSON(&payload) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } rawKey, err := base64.StdEncoding.DecodeString(payload.Key) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // call API pubkey, apiErr := caliopen.Facilities.RESTfacility.CreatePGPPubKey(payload.Label, rawKey, contact) if apiErr != nil { returnedErr := new(swgErr.CompositeError) switch apiErr.Code() { case UnprocessableCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusUnprocessableEntity, "api returned unprocessable error"), apiErr, apiErr.Cause()) case DbCaliopenErr: if prevErr, ok := apiErr.Cause().(CaliopenError); ok { switch prevErr.Code() { case ForbiddenCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusForbidden, "api returned forbidden error"), apiErr, apiErr.Cause()) case NotFoundCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "api failed to retrieve in store"), apiErr, apiErr.Cause()) default: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), apiErr, apiErr.Cause()) } } else { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), apiErr, apiErr.Cause()) } default: returnedErr = swgErr.CompositeValidationError(apiErr, apiErr.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() } else { ctx.JSON(http.StatusOK, struct { Location string `json:"location"` PublicKeyID string `json:"publickey_id"` }{ http_middleware.RoutePrefix + http_middleware.ContactsRoute + "/" + contactId + "/publickeys/" + pubkey.KeyId.String(), pubkey.KeyId.String(), }) } return } // GetPubKeys handles GET …/contacts/:contactID/publickeys func GetPubKeys(ctx *gin.Context) { // check payload userId := ctx.MustGet("user_id").(string) userId, err := operations.NormalizeUUIDstring(userId) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } contactId := ctx.Param("contactID") if contactId == "" { e := swgErr.New(http.StatusUnprocessableEntity, "empty contactID") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } contactId, err = operations.NormalizeUUIDstring(contactId) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // check if contact exist to return relevant error if !caliopen.Facilities.RESTfacility.ContactExists(userId, contactId) { e := swgErr.New(http.StatusNotFound, "contact not found") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // call API keys, apiErr := caliopen.Facilities.RESTfacility.RetrieveContactPubKeys(userId, contactId) if apiErr != nil { returnedErr := new(swgErr.CompositeError) switch apiErr.Code() { case UnprocessableCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusUnprocessableEntity, "api returned unprocessable error"), apiErr, apiErr.Cause()) case DbCaliopenErr: if prevErr, ok := apiErr.Cause().(CaliopenError); ok { switch prevErr.Code() { case ForbiddenCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusForbidden, "api returned forbidden error"), apiErr, apiErr.Cause()) case NotFoundCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "api failed to retrieve in store"), apiErr, apiErr.Cause()) default: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), apiErr, apiErr.Cause()) } } else { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), apiErr, apiErr.Cause()) } default: returnedErr = swgErr.CompositeValidationError(apiErr, apiErr.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() } else { var respBuf bytes.Buffer respBuf.WriteString("{\"total\": " + strconv.Itoa(len(keys)) + ", ") respBuf.WriteString("\"pubkeys\":[") first := true for _, pubkey := range keys { jsonKey, err := pubkey.MarshalFrontEnd() if err == nil { if first { first = false } else { respBuf.WriteByte(',') } respBuf.Write(jsonKey) } } respBuf.WriteString("]}") ctx.Data(http.StatusOK, "application/json; charset=utf-8", respBuf.Bytes()) } } // GetPubKey handles GET …/contacts/:contactID/publickeys/:pubkeyID func GetPubKey(ctx *gin.Context) { // check payload userId := ctx.MustGet("user_id").(string) userId, err1 := operations.NormalizeUUIDstring(userId) contactId := ctx.Param("contactID") contactId, err2 := operations.NormalizeUUIDstring(contactId) pubkeyId := ctx.Param("pubkeyID") pubkeyId, err3 := operations.NormalizeUUIDstring(pubkeyId) if err1 != nil || err2 != nil || err3 != nil { e := swgErr.New(http.StatusUnprocessableEntity, "invalid uuid") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // check if contact exist to return relevant error if !caliopen.Facilities.RESTfacility.ContactExists(userId, contactId) { e := swgErr.New(http.StatusNotFound, "contact not found") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // call API pubkey, err := caliopen.Facilities.RESTfacility.RetrievePubKey(userId, contactId, pubkeyId) if err != nil { returnedErr := new(swgErr.CompositeError) switch err.Code() { case UnprocessableCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusUnprocessableEntity, "api returned unprocessable error"), err, err.Cause()) case DbCaliopenErr: if prevErr, ok := err.Cause().(CaliopenError); ok { switch prevErr.Code() { case ForbiddenCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusForbidden, "api returned forbidden error"), err, err.Cause()) case NotFoundCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "api failed to retrieve in store"), err, err.Cause()) default: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), err, err.Cause()) } } else { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), err, err.Cause()) } default: returnedErr = swgErr.CompositeValidationError(err, err.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } key_json, e := pubkey.MarshalFrontEnd() if e != nil { se := swgErr.New(http.StatusFailedDependency, e.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, se) ctx.Abort() } else { ctx.Data(http.StatusOK, "application/json; charset=utf-8", key_json) } } // PatchPubKey handles PATCH …/contacts/:contactID/publickeys/:pubkeyID func PatchPubKey(ctx *gin.Context) { // check payload userId := ctx.MustGet("user_id").(string) userId, err1 := operations.NormalizeUUIDstring(userId) contactId := ctx.Param("contactID") contactId, err2 := operations.NormalizeUUIDstring(contactId) pubkeyId := ctx.Param("pubkeyID") pubkeyId, err3 := operations.NormalizeUUIDstring(pubkeyId) if err1 != nil || err2 != nil || err3 != nil { e := swgErr.New(http.StatusUnprocessableEntity, "invalid uuid") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } var patch []byte patch, err := ioutil.ReadAll(ctx.Request.Body) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // check if contact exist to return relevant error if !caliopen.Facilities.RESTfacility.ContactExists(userId, contactId) { e := swgErr.New(http.StatusNotFound, "contact not found") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } caliopenErr := caliopen.Facilities.RESTfacility.PatchPubKey(patch, userId, contactId, pubkeyId) if caliopenErr != nil { returnedErr := new(swgErr.CompositeError) switch caliopenErr.Code() { case UnprocessableCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusUnprocessableEntity, "api returned unprocessable error"), caliopenErr, caliopenErr.Cause()) case DbCaliopenErr: if prevErr, ok := caliopenErr.Cause().(CaliopenError); ok { switch prevErr.Code() { case ForbiddenCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusForbidden, "api returned forbidden error"), caliopenErr, caliopenErr.Cause()) case NotFoundCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "api failed to retrieve in store"), caliopenErr, caliopenErr.Cause()) default: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), caliopenErr, caliopenErr.Cause()) } } else { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), caliopenErr, caliopenErr.Cause()) } default: returnedErr = swgErr.CompositeValidationError(caliopenErr, caliopenErr.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } else { ctx.Status(http.StatusNoContent) } } // DeletePubKey handles DELETE …/contacts/:contactID/publickeys/:pubkeyID func DeletePubKey(ctx *gin.Context) { // check payload userId := ctx.MustGet("user_id").(string) userId, err1 := operations.NormalizeUUIDstring(userId) contactId := ctx.Param("contactID") contactId, err2 := operations.NormalizeUUIDstring(contactId) pubkeyId := ctx.Param("pubkeyID") pubkeyId, err3 := operations.NormalizeUUIDstring(pubkeyId) if err1 != nil || err2 != nil || err3 != nil { e := swgErr.New(http.StatusUnprocessableEntity, "invalid uuid") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // check if contact exist to return relevant error if !caliopen.Facilities.RESTfacility.ContactExists(userId, contactId) { e := swgErr.New(http.StatusNotFound, "contact not found") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // call API pubkey := &PublicKey{ KeyId: UUID(uuid.FromStringOrNil(pubkeyId)), ResourceId: UUID(uuid.FromStringOrNil(contactId)), UserId: UUID(uuid.FromStringOrNil(userId)), } err := caliopen.Facilities.RESTfacility.DeletePubKey(pubkey) if err != nil { returnedErr := new(swgErr.CompositeError) switch err.Code() { case UnprocessableCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusUnprocessableEntity, "api returned unprocessable error"), err, err.Cause()) case DbCaliopenErr: if prevErr, ok := err.Cause().(CaliopenError); ok { switch prevErr.Code() { case ForbiddenCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusForbidden, "api returned forbidden error"), err, err.Cause()) case NotFoundCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "api failed to retrieve in store"), err, err.Cause()) default: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), err, err.Cause()) } } else { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), err, err.Cause()) } default: returnedErr = swgErr.CompositeValidationError(err, err.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() } else { ctx.Status(http.StatusNoContent) } } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/devices/devices.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package devices import ( "bytes" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" log "github.com/Sirupsen/logrus" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "io/ioutil" "net/http" "strconv" ) // NewDevice handles POST /devices func NewDevice(ctx *gin.Context) { e := swgErr.New(http.StatusForbidden, "do not create new device this way") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return /* LEGACY code var device Device b := binding.JSON if err := b.Bind(ctx.Request, &device); err == nil { user_uuid, _ := uuid.FromString(ctx.MustGet("user_id").(string)) device.UserId.UnmarshalBinary(user_uuid.Bytes()) if device.Name == "" || strings.Replace(device.Name, " ", "", -1) == "" { err := errors.New("device's name is empty") e := swgErr.New(http.StatusBadRequest, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } if device.Type == "" || strings.Replace(device.Type, " ", "", -1) == "" { err := errors.New("device's type is empty") e := swgErr.New(http.StatusBadRequest, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } device.Name = strings.TrimSpace(device.Name) device.Type = strings.TrimSpace(device.Type) device.IpCreation = ctx.ClientIP() device.UserAgent = ctx.GetHeader("User-Agent") err := caliopen.Facilities.RESTfacility.CreateDevice(&device) if err != nil { returnedErr := new(swgErr.CompositeError) if err.Code() == DbCaliopenErr && err.Cause().Error() == "not found" { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), err, err.Cause()) } else { returnedErr = swgErr.CompositeValidationError(err, err.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() } else { ctx.JSON(http.StatusOK, struct { Location string `json:"location"` DeviceId string `json:"device_id"` }{ http_middleware.RoutePrefix + http_middleware.DevicesRoute + "/" + device.DeviceId.String(), device.DeviceId.String(), }) } } else { e := swgErr.New(http.StatusBadRequest, fmt.Sprintf("Unable to json marshal the provided payload : %s", err)) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } */ } // GetDevicesList handles GET /devices func GetDevicesList(ctx *gin.Context) { userId := ctx.MustGet("user_id").(string) devices, err := caliopen.Facilities.RESTfacility.RetrieveDevices(userId) if err != nil && err.Cause().Error() != "devices not found" { returnedErr := new(swgErr.CompositeError) returnedErr = swgErr.CompositeValidationError(err, err.Cause()) http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } var respBuf bytes.Buffer respBuf.WriteString("{\"total\": " + strconv.Itoa(len(devices)) + ",") respBuf.WriteString(("\"devices\":[")) first := true for _, device := range devices { json_device, err := device.MarshalFrontEnd() if err == nil { if first { first = false } else { respBuf.WriteByte(',') } respBuf.Write(json_device) } } respBuf.WriteString("]}") ctx.Data(http.StatusOK, "application/json; charset=utf-8", respBuf.Bytes()) } // GetDevice handles GET /devices/:deviceID func GetDevice(ctx *gin.Context) { userId := ctx.MustGet("user_id").(string) deviceId, err := operations.NormalizeUUIDstring(ctx.Param("deviceID")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } device, CalErr := caliopen.Facilities.RESTfacility.RetrieveDevice(userId, deviceId) if CalErr != nil { returnedErr := new(swgErr.CompositeError) if CalErr.Code() == DbCaliopenErr && CalErr.Cause().Error() == "not found" { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), CalErr, CalErr.Cause()) } else { returnedErr = swgErr.CompositeValidationError(CalErr, CalErr.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } device_json, err := device.MarshalFrontEnd() if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Data(http.StatusOK, "application/json; charset=utf-8", device_json) } } // PatchDevice handles PATCH /devices/:deviceID func PatchDevice(ctx *gin.Context) { var err error userId, err := operations.NormalizeUUIDstring(ctx.MustGet("user_id").(string)) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } deviceId, err := operations.NormalizeUUIDstring(ctx.Param("deviceID")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } var patch []byte patch, err = ioutil.ReadAll(ctx.Request.Body) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // call REST facility with payload err = caliopen.Facilities.RESTfacility.PatchDevice(patch, userId, deviceId) if err != nil { if Cerr, ok := err.(CaliopenError); ok { returnedErr := new(swgErr.CompositeError) if Cerr.Code() == DbCaliopenErr && Cerr.Cause().Error() == "not found" { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), Cerr, Cerr.Cause()) } else { returnedErr = swgErr.CompositeValidationError(Cerr, Cerr.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Status(http.StatusNoContent) } } // DeleteDevice handles DELETE /devices/:deviceID func DeleteDevice(ctx *gin.Context) { var err error userId, err := operations.NormalizeUUIDstring(ctx.MustGet("user_id").(string)) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } deviceId, err := operations.NormalizeUUIDstring(ctx.Param("deviceID")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } err = caliopen.Facilities.RESTfacility.DeleteDevice(userId, deviceId) if err != nil { if Cerr, ok := err.(CaliopenError); ok { returnedErr := new(swgErr.CompositeError) if Cerr.Code() == DbCaliopenErr && Cerr.Cause().Error() == "not found" { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), Cerr, Cerr.Cause()) } else { returnedErr = swgErr.CompositeValidationError(Cerr, Cerr.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Status(http.StatusNoContent) } } // Actions handles POST /devices/:deviceID/actions func Actions(ctx *gin.Context) { var err error userId, err := operations.NormalizeUUIDstring(ctx.MustGet("user_id").(string)) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } deviceId, err := operations.NormalizeUUIDstring(ctx.Param("deviceID")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } var actions ActionsPayload if err := ctx.BindJSON(&actions); err == nil { switch actions.Actions[0] { case "device-validation": // validate params in payload before calling API valid := true var channel string if actions.Params == nil { valid = false } else if params, ok := actions.Params.(map[string]interface{}); ok { if channel, ok = params["channel"].(string); channel == "" || !ok { valid = false } } else { valid = false } if !valid { e := swgErr.New(http.StatusUnprocessableEntity, "params is missing or malformed") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } actions.UserId = userId actions.Params.(map[string]interface{})["device_id"] = deviceId err := caliopen.Facilities.RESTfacility.RequestDeviceValidation(userId, deviceId, channel, caliopen.Facilities.Notifiers) if err != nil { if Cerr, ok := err.(CaliopenError); ok { returnedErr := new(swgErr.CompositeError) if (Cerr.Code() == DbCaliopenErr && Cerr.Cause().Error() == "not found") || Cerr.Code() == NotFoundCaliopenErr { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), Cerr, Cerr.Cause()) } else { returnedErr = swgErr.CompositeValidationError(Cerr, Cerr.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Status(http.StatusNoContent) } default: e := swgErr.New(http.StatusNotImplemented, "unknown action "+actions.Actions[0]) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } } else { log.WithError(err).Errorf("failed to bind json payload to ActionsPayload struct") e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } } // ValidateDevice handles GET /validate-device/:token func ValidateDevice(ctx *gin.Context) { var err error userId, err := operations.NormalizeUUIDstring(ctx.MustGet("user_id").(string)) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } token := ctx.Param("token") if token == "" { e := swgErr.New(http.StatusUnprocessableEntity, "validation token is empty") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } err = caliopen.Facilities.RESTfacility.ConfirmDeviceValidation(userId, token) if err != nil { e := swgErr.New(http.StatusNotFound, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Status(http.StatusNoContent) } return } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/discussions/discussions.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package discussions import ( "bytes" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "net/http" "strconv" ) // GET …/discussions func GetDiscussionsList(ctx *gin.Context) { // temporary hack to check if X-Caliopen-IL header is in request, because go-openapi pkg fails to do it. // (NB : CanonicalHeaderKey func normalize http headers with uppercase at beginning of words) if _, ok := ctx.Request.Header["X-Caliopen-Il"]; !ok { e := swgErr.New(http.StatusFailedDependency, "Missing mandatory header 'X-Caliopen-Il'.") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } ILrange := operations.GetImportanceLevel(ctx) // temporary hack to check if X-Caliopen-IL header is in request, because go-openapi pkg fails to do it. // (NB : CanonicalHeaderKey func normalize http headers with uppercase at beginning of words) if _, ok := ctx.Request.Header["X-Caliopen-Pi"]; !ok { e := swgErr.New(http.StatusFailedDependency, "Missing mandatory header 'X-Caliopen-PI'.") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } PIrange := operations.GetPrivacyIndex(ctx) userId, err := operations.NormalizeUUIDstring(ctx.GetString("user_id")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } shardId := ctx.MustGet("shard_id").(string) userInfo := &UserInfo{User_id: userId, Shard_id: shardId} var limit, offset int query_values := ctx.Request.URL.Query() if l, ok := query_values["limit"]; ok { limit, _ = strconv.Atoi(l[0]) query_values.Del("limit") } if o, ok := query_values["offset"]; ok { offset, _ = strconv.Atoi(o[0]) query_values.Del("offset") } list, totalFound, err := caliopen.Facilities.RESTfacility.GetDiscussionsList(userInfo, ILrange, PIrange, limit, offset) if err != nil && err.Error() != "not found" { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } var respBuf bytes.Buffer respBuf.WriteString("{\"total\": " + strconv.FormatInt(int64(totalFound), 10) + ",") respBuf.WriteString("\"discussions\":[") first := true for _, disc := range list { json_disc, err := disc.MarshalFrontEnd() if err == nil { if first { first = false } else { respBuf.WriteByte(',') } respBuf.Write(json_disc) } } respBuf.WriteString("]}") ctx.Data(http.StatusOK, "application/json; charset=utf-8", respBuf.Bytes()) } // GET …/discussions/:discussionId func GetDiscussion(ctx *gin.Context) { userId, err := operations.NormalizeUUIDstring(ctx.GetString("user_id")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } shardId := ctx.MustGet("shard_id").(string) userInfo := &UserInfo{User_id: userId, Shard_id: shardId} discussion, err := caliopen.Facilities.RESTfacility.DiscussionMetadata(userInfo, ctx.Param("discussionId")) if err != nil { var e error if err.Error() == "not found" { e = swgErr.New(http.StatusNotFound, err.Error()) } else { e = swgErr.New(http.StatusFailedDependency, err.Error()) } http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } disc_json, err := discussion.MarshalFrontEnd() if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Data(http.StatusOK, "application/json; charset=utf-8", disc_json) } } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/helpers.go ================================================ package operations import ( "github.com/gin-gonic/gin" "github.com/satori/go.uuid" "strconv" "strings" ) // fall back to default values if can't extract valid numbers. func GetImportanceLevel(ctx *gin.Context) (il [2]int8) { il = [2]int8{-10, 10} // default values var from, to int var err error if il_header, ok := ctx.Request.Header["X-Caliopen-Il"]; !ok { return il } else { il_range_str := strings.Split(il_header[0], ";") // get only first value found if len(il_range_str) != 2 { return il } from, err = strconv.Atoi(il_range_str[0]) if err != nil { from = -10 } to, err = strconv.Atoi(il_range_str[1]) if err != nil { to = 10 } if from < -10 || from > 10 { from = -10 } if to < -10 || to > 10 { to = 10 } if from > to { from = to } il[0] = int8(from) il[1] = int8(to) return } } // fall back to default values if can't extract valid numbers. func GetPrivacyIndex(ctx *gin.Context) (pi [2]int8) { pi = [2]int8{0, 100} // default values var from, to int var err error if pi_header, ok := ctx.Request.Header["X-Caliopen-Pi"]; !ok { return pi } else { pi_range_str := strings.Split(pi_header[0], ";") // get only first value found if len(pi_range_str) != 2 { return pi } from, err = strconv.Atoi(pi_range_str[0]) if err != nil { from = 0 } to, err = strconv.Atoi(pi_range_str[1]) if err != nil { to = 100 } if from < 0 || from > 100 { from = 0 } if to < 0 || to > 100 { to = 100 } if from > to { from = to } pi[0] = int8(from) pi[1] = int8(to) return } } // NormalizeUUIDstring returns a valid uuidv4 string from input // or an error if input string is invalid. // Following input formats are supported: // "6ba7b810-9dad-11d1-80b4-00c04fd430c8", // "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}", // "urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8" // Output string is always in "6ba7b810-9dad-11d1-80b4-00c04fd430c8" format. func NormalizeUUIDstring(uuid_str string) (string, error) { id, err := uuid.FromString(uuid_str) if err != nil { return "", err } return id.String(), nil } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/identities/identities.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package identities import ( "bytes" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "github.com/satori/go.uuid" "io/ioutil" "net/http" "strconv" "strings" ) //GET …/identities/locals/{identity_id} func GetLocalIdentity(ctx *gin.Context) { e := swgErr.New(http.StatusNotImplemented, "not implemented") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } //GET …/identities/locals func GetLocalsIdentities(ctx *gin.Context) { user_id := ctx.MustGet("user_id").(string) identities, err := caliopen.Facilities.RESTfacility.RetrieveLocalIdentities(user_id) if err != nil && err.Error() != "not found" { e := swgErr.New(http.StatusInternalServerError, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } ret := struct { Total int `json:"total"` LocalsIdentities []UserIdentity `json:"local_identities"` }{len(identities), identities} ctx.JSON(http.StatusOK, ret) } // GetRemoteIdentities handles GET …/identities/remotes func GetRemoteIdentities(ctx *gin.Context) { userID, err := operations.NormalizeUUIDstring(ctx.GetString("user_id")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } noCredentials := false // by default do not return Credentials list, e := caliopen.Facilities.RESTfacility.RetrieveRemoteIdentities(userID, noCredentials) if e != nil && e.Code() != NotFoundCaliopenErr { returnedErr := new(swgErr.CompositeError) returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "RESTfacility returned error"), e, e.Cause()) http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } var respBuf bytes.Buffer respBuf.WriteString("{\"total\": " + strconv.Itoa(len(list)) + ",") respBuf.WriteString("\"remote_identities\":[") first := true for _, id := range list { json_id, err := id.MarshalFrontEnd() if err == nil { if first { first = false } else { respBuf.WriteByte(',') } respBuf.Write(json_id) } } respBuf.WriteString("]}") ctx.Data(http.StatusOK, "application/json; charset=utf-8", respBuf.Bytes()) } // GetRemoteIdentity handles GET …/identities/remotes/:remote_id func GetRemoteIdentity(ctx *gin.Context) { userID, err := operations.NormalizeUUIDstring(ctx.GetString("user_id")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } remote_id := ctx.Param("remote_id") if remote_id == "" { e := swgErr.New(http.StatusUnprocessableEntity, "empty remote_id") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } withCredentials := false // by default do not return Credentials identity, e := caliopen.Facilities.RESTfacility.RetrieveUserIdentity(userID, remote_id, withCredentials) if e != nil { returnedErr := new(swgErr.CompositeError) if e.Code() == NotFoundCaliopenErr { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), e, e.Cause()) } else { returnedErr = swgErr.CompositeValidationError(e, e.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() } else { if identity.Type != RemoteIdentity { e := swgErr.New(http.StatusNotFound, "resource not available on this route") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } id_json, err := identity.MarshalFrontEnd() if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Data(http.StatusOK, "application/json; charset=utf-8", id_json) } } } // NewRemoteIdentity handles POST …/identities/remotes func NewRemoteIdentity(ctx *gin.Context) { // check payload user_id := ctx.MustGet("user_id").(string) userID, err := operations.NormalizeUUIDstring(user_id) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } identity := new(UserIdentity) identity.MarshallNew() err = ctx.ShouldBindJSON(identity) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } if identity.Protocol == "" { e := swgErr.New(http.StatusUnprocessableEntity, "mandatory property `protocol` is missing") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } if identity.Identifier == "" { e := swgErr.New(http.StatusUnprocessableEntity, "mandatory property `identifier` is missing") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } identity.Identifier = strings.ToLower(identity.Identifier) // add UserId and type identity.UserId.UnmarshalBinary(uuid.FromStringOrNil(userID).Bytes()) identity.Type = RemoteIdentity // call api apiErr := caliopen.Facilities.RESTfacility.CreateUserIdentity(identity) if apiErr != nil { returnedErr := new(swgErr.CompositeError) switch apiErr.Code() { case UnprocessableCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusUnprocessableEntity, "api returned unprocessable error"), apiErr, apiErr.Cause()) case DbCaliopenErr: if prevErr, ok := apiErr.Cause().(CaliopenError); ok { switch prevErr.Code() { case ForbiddenCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusForbidden, "api returned forbidden error"), apiErr, apiErr.Cause()) case NotFoundCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "api failed to retrieve in store"), apiErr, apiErr.Cause()) default: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), apiErr, apiErr.Cause()) } } else { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), apiErr, apiErr.Cause()) } default: returnedErr = swgErr.CompositeValidationError(apiErr, apiErr.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() } else { ctx.JSON(http.StatusOK, struct { Location string `json:"location"` RemoteId string `json:"remote_id"` }{ http_middleware.RoutePrefix + http_middleware.IdentitiesRoute + "/remotes/" + identity.Id.String(), identity.Id.String(), }) } return } // PatchRemoteIdentity handles PATCH …/identities/remotes/:remote_id func PatchRemoteIdentity(ctx *gin.Context) { var err error userId, err := operations.NormalizeUUIDstring(ctx.MustGet("user_id").(string)) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } remoteId := ctx.Param("remote_id") if remoteId == "" { e := swgErr.New(http.StatusBadRequest, "empty remote_id") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } var patch []byte patch, err = ioutil.ReadAll(ctx.Request.Body) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } if !caliopen.Facilities.RESTfacility.IsRemoteIdentity(userId, remoteId) { e := swgErr.New(http.StatusNotFound, "resource not found on this route") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // call REST facility with payload apiErr := caliopen.Facilities.RESTfacility.PatchUserIdentity(patch, userId, remoteId) if apiErr != nil { returnedErr := new(swgErr.CompositeError) switch apiErr.Code() { case UnprocessableCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusUnprocessableEntity, "api returned unprocessable error"), apiErr, apiErr.Cause()) case DbCaliopenErr: if prevErr, ok := apiErr.Cause().(CaliopenError); ok { switch prevErr.Code() { case ForbiddenCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusForbidden, "api returned forbidden error"), apiErr, apiErr.Cause()) case NotFoundCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "api failed to retrieve in store"), apiErr, apiErr.Cause()) default: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), apiErr, apiErr.Cause()) } } else { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), apiErr, apiErr.Cause()) } default: returnedErr = swgErr.CompositeValidationError(apiErr, apiErr.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() } else { ctx.Status(http.StatusNoContent) } } // DeleteRemoteIdentity handles DELETE …/identities/remotes/:remote_id func DeleteRemoteIdentity(ctx *gin.Context) { var err error // check request userId, err := operations.NormalizeUUIDstring(ctx.MustGet("user_id").(string)) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } remoteId := ctx.Param("remote_id") if remoteId == "" { e := swgErr.New(http.StatusBadRequest, "empty remote_id") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } if !caliopen.Facilities.RESTfacility.IsRemoteIdentity(userId, remoteId) { e := swgErr.New(http.StatusNotFound, "resource not found on this route") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // call api apiErr := caliopen.Facilities.RESTfacility.DeleteUserIdentity(userId, remoteId) if apiErr != nil { returnedErr := new(swgErr.CompositeError) switch apiErr.Code() { case UnprocessableCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusUnprocessableEntity, "api returned unprocessable error"), apiErr, apiErr.Cause()) case DbCaliopenErr: if prevErr, ok := apiErr.Cause().(CaliopenError); ok { switch prevErr.Code() { case ForbiddenCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusForbidden, "api returned forbidden error"), apiErr, apiErr.Cause()) case NotFoundCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "api failed to retrieve in store"), apiErr, apiErr.Cause()) default: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), apiErr, apiErr.Cause()) } } else { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "api failed to call store"), apiErr, apiErr.Cause()) } default: returnedErr = swgErr.CompositeValidationError(apiErr, apiErr.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() } else { ctx.Status(http.StatusNoContent) } } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/imports/import.go ================================================ /* * // Copyleft (ɔ) 2019 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package imports import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "net/http" ) // ImportFile handles POST /imports and do logic depending on file mime type func ImportFile(ctx *gin.Context) { userId, err := operations.NormalizeUUIDstring(ctx.MustGet("user_id").(string)) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } shard_id := ctx.MustGet("shard_id").(string) user_info := &UserInfo{User_id: userId, Shard_id: shard_id} file, _, err := ctx.Request.FormFile("file") if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // TODO manage import switch file content-type / name err = caliopen.Facilities.RESTfacility.ImportVcardFile(user_info, file) if err != nil { var e error e = swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Status(http.StatusOK) } return } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/messages/actions.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package messages import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" log "github.com/Sirupsen/logrus" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "net/http" ) // POST …/:message_id/actions func Actions(ctx *gin.Context) { user_id := ctx.MustGet("user_id").(string) shard_id := ctx.MustGet("shard_id").(string) user_info := &UserInfo{User_id: user_id, Shard_id: shard_id} msg_id, err := operations.NormalizeUUIDstring(ctx.Param("message_id")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } var actions ActionsPayload if err := ctx.BindJSON(&actions); err == nil { switch actions.Actions[0] { case "send": updated_msg, err := caliopen.Facilities.RESTfacility.SendDraft(user_info, msg_id) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { // TODO : find the correct body_type to use msg_json, err := updated_msg.MarshalFrontEnd("plain_text") if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Data(http.StatusOK, "application/json; charset=utf-8", msg_json) } } case "set_read": err := caliopen.Facilities.RESTfacility.SetMessageUnread(user_info, msg_id, false) if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Status(http.StatusNoContent) } case "set_unread": err := caliopen.Facilities.RESTfacility.SetMessageUnread(user_info, msg_id, true) if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Status(http.StatusNoContent) } default: e := swgErr.New(http.StatusNotImplemented, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } } else { log.WithError(err).Errorf("failed to bind json payload to ActionsPayload struct") e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/messages/attachments.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package messages import ( "bytes" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "mime" "net/http" "strconv" "time" ) // POST …/:message_id/attachments func UploadAttachment(ctx *gin.Context) { user_id := ctx.MustGet("user_id").(string) shard_id := ctx.MustGet("shard_id").(string) user := &UserInfo{User_id: user_id, Shard_id: shard_id} msg_id, err := operations.NormalizeUUIDstring(ctx.Param("message_id")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } file, header, err := ctx.Request.FormFile("attachment") if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } filename := header.Filename content_type := header.Header["Content-Type"][0] attchmtUrl, err := caliopen.Facilities.RESTfacility.AddAttachment(user, msg_id, filename, content_type, file) if err != nil { var e error if err.Error() == "not found" { e = swgErr.New(http.StatusNotFound, err.Error()) } else { e = swgErr.New(http.StatusFailedDependency, err.Error()) } http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } resp := struct { TempId string `json:"temp_id"` }{attchmtUrl} ctx.JSON(http.StatusOK, resp) } // DELETE …/:message_id/attachments/:attachment_id func DeleteAttachment(ctx *gin.Context) { user_id := ctx.MustGet("user_id").(string) shard_id := ctx.MustGet("shard_id").(string) user := &UserInfo{User_id: user_id, Shard_id: shard_id} msg_id, err := operations.NormalizeUUIDstring(ctx.Param("message_id")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } attch_id, err := operations.NormalizeUUIDstring(ctx.Param("attachment_id")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } caliopenErr := caliopen.Facilities.RESTfacility.DeleteAttachment(user, msg_id, attch_id) if caliopenErr != nil { returnedErr := new(swgErr.CompositeError) if caliopenErr.Error() == "message not found" || caliopenErr.Error() == "attachment not found" || caliopenErr.Code() == NotFoundCaliopenErr { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), caliopenErr, caliopenErr.Cause()) } else { returnedErr = swgErr.CompositeValidationError(caliopenErr, caliopenErr.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) } ctx.Status(http.StatusOK) } // GET …/:message_id/attachments/:attachment_id // sends attachment as a file to client func DownloadAttachment(ctx *gin.Context) { user_id := ctx.MustGet("user_id").(string) msg_id, err := operations.NormalizeUUIDstring(ctx.Param("message_id")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } meta, content, err := caliopen.Facilities.RESTfacility.OpenAttachment(user_id, msg_id, ctx.Param("attachment_id")) if err != nil { var e error if err.Error() == "attachment not found" { e = swgErr.New(http.StatusNotFound, err.Error()) } else { e = swgErr.New(http.StatusFailedDependency, err.Error()) } http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // create a ReaderSeeker from the io.Reader returned by OpenAttachment size, err := strconv.ParseInt(meta["Message-Size"], 10, 64) if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } attch_bytes := make([]byte, size) content.Read(attch_bytes) rs := bytes.NewReader(attch_bytes) ctx.Header("Content-Type", meta["Content-Type"]) ctx.Header("Content-Disposition", `attachment; filename="`+mime.BEncoding.Encode("UTF-8", meta["Filename"])+`"`) http.ServeContent(ctx.Writer, ctx.Request, "", time.Time{}, rs) } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/messages/messages.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package messages import ( "bytes" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/pi" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "github.com/satori/go.uuid" "net/http" "strconv" ) // GET …/messages func GetMessagesList(ctx *gin.Context) { // temporary hack to check if X-Caliopen-IL header is in request, because go-openapi pkg fails to do it. // (NB : CanonicalHeaderKey func normalize http headers with uppercase at beginning of words) if _, ok := ctx.Request.Header["X-Caliopen-Il"]; !ok { e := swgErr.New(http.StatusFailedDependency, "Missing mandatory header 'X-Caliopen-Il'.") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } var limit, offset int var user_UUID UUID user_uuid_str := ctx.MustGet("user_id").(string) shard_id := ctx.MustGet("shard_id").(string) user_uuid, _ := uuid.FromString(user_uuid_str) user_UUID.UnmarshalBinary(user_uuid.Bytes()) query_values := ctx.Request.URL.Query() if l, ok := query_values["limit"]; ok { limit, _ = strconv.Atoi(l[0]) query_values.Del("limit") } if o, ok := query_values["offset"]; ok { offset, _ = strconv.Atoi(o[0]) query_values.Del("offset") } // check for params to retrieve a range of messages 'around' a specific one // params `msg_id` and `range[]` must be both present var msgID string var msgRange []string if id, ok := query_values["msg_id"]; ok { msgID = id[0] } msgRange = query_values["range[]"] if msgID != "" && len(msgRange) == 0 { e := swgErr.New(http.StatusBadRequest, "range[] param must be provided with msg_id param") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } if msgID == "" && len(msgRange) != 0 { e := swgErr.New(http.StatusBadRequest, "msg_id param must be provided with range[] param") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } filter := IndexSearch{ Shard_id: shard_id, User_id: user_UUID, Terms: map[string][]string(query_values), Limit: limit, Offset: offset, ILrange: operations.GetImportanceLevel(ctx), } var list []*Message var totalFound int64 var err error if msgID != "" && len(msgRange) != 0 { list, totalFound, err = caliopen.Facilities.RESTfacility.GetMessagesRange(filter) } else { list, totalFound, err = caliopen.Facilities.RESTfacility.GetMessagesList(filter) } if err != nil && err.Error() != "not found" { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } settings, err := caliopen.Facilities.RESTfacility.GetSettings(user_uuid_str) if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } var respBuf bytes.Buffer respBuf.WriteString("{\"total\": " + strconv.FormatInt(totalFound, 10) + ",") respBuf.WriteString("\"messages\":[") first := true for _, msg := range list { msg.PI = pi.ComputePIMessage(msg) json_msg, err := msg.MarshalFrontEnd(settings.MessageDisplayFormat) if err == nil { if first { first = false } else { respBuf.WriteByte(',') } respBuf.Write(json_msg) } } respBuf.WriteString("]}") ctx.Data(http.StatusOK, "application/json; charset=utf-8", respBuf.Bytes()) } // GET …/messages/:message_id func GetMessage(ctx *gin.Context) { user_id := ctx.MustGet("user_id").(string) shard_id := ctx.MustGet("shard_id").(string) user_info := &UserInfo{User_id: user_id, Shard_id: shard_id} msg_id, err := operations.NormalizeUUIDstring(ctx.Param("message_id")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } msg, err := caliopen.Facilities.RESTfacility.GetMessage(user_info, msg_id) if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } settings, err := caliopen.Facilities.RESTfacility.GetSettings(user_id) if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } msg.PI = pi.ComputePIMessage(msg) msg_json, err := msg.MarshalFrontEnd(settings.MessageDisplayFormat) if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Data(http.StatusOK, "application/json; charset=utf-8", msg_json) } } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/notifications/notifications.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package notifications import ( "bytes" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "github.com/satori/go.uuid" "net/http" "regexp" "strconv" "time" ) // GetPendingNotif handles GET /notifications // two optional params may be in query : `to` and `from` to narrow the notifications list to either : // - a time range // - a uuid range func GetPendingNotif(ctx *gin.Context) { userId := ctx.MustGet("user_id").(string) to_param := ctx.Query("to") from_param := ctx.Query("from") var to_kind, from_kind string uuidv1Regex := regexp.MustCompile(`[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-1[0-9a-fA-F]{3}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}`) rfc3339Regex := regexp.MustCompile(`(?i)^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(Z|(\+|-)([01][0-9]|2[0-3]):([0-5][0-9]))$`) // validate from param if from_param != "" { if uuidv1Regex.MatchString(from_param) { from_kind = "id" } else if rfc3339Regex.MatchString(from_param) { from_kind = "time" } if from_kind == "" { e := swgErr.New(http.StatusUnprocessableEntity, "invalid from param") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } } // validate to param if to_param != "" { if uuidv1Regex.MatchString(to_param) { to_kind = "id" } else if rfc3339Regex.MatchString(to_param) { to_kind = "time" } if to_kind == "" { e := swgErr.New(http.StatusUnprocessableEntity, "invalid to param") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } } // validate params consistency if (from_param != "" && to_param != "") && (from_kind != to_kind) { e := swgErr.New(http.StatusUnprocessableEntity, "params are mix of time and uuid") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } var notifs []Notification var err CaliopenError // call relevant API if from_kind == "time" || to_kind == "time" || (from_param == "" && to_param == "") { from, to := time.Time{}, time.Time{} if to_param != "" { t, err := time.Parse(time.RFC3339, to_param) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } to = t } if from_param != "" { f, err := time.Parse(time.RFC3339, from_param) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } from = f } notifs, err = caliopen.Facilities.Notifiers.NotificationsByTime(userId, from, to) if err != nil && err.Cause().Error() != "notifications not found" { returnedErr := new(swgErr.CompositeError) returnedErr = swgErr.CompositeValidationError(err, err.Cause()) http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() } } if from_kind == "id" || to_kind == "id" { var from, to string if to_param != "" { t, err := uuid.FromString(to_param) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } to = t.String() } if from_param != "" { f, err := uuid.FromString(from_param) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } from = f.String() } notifs, err = caliopen.Facilities.Notifiers.NotificationsByID(userId, from, to) if err != nil && err.Cause().Error() != "notifications not found" { returnedErr := new(swgErr.CompositeError) returnedErr = swgErr.CompositeValidationError(err, err.Cause()) http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() } } var respBuf bytes.Buffer respBuf.WriteString("{\"total\": " + strconv.Itoa(len(notifs)) + ",") respBuf.WriteString(("\"notifications\":[")) first := true for _, notif := range notifs { json_notif, err := notif.MarshalFrontEnd() if err == nil { if first { first = false } else { respBuf.WriteByte(',') } respBuf.Write(json_notif) } } respBuf.WriteString("]}") ctx.Data(http.StatusOK, "application/json; charset=utf-8", respBuf.Bytes()) } // DeleteNotifications handles DELETE /notifications func DeleteNotifications(ctx *gin.Context) { userId := ctx.MustGet("user_id").(string) until := time.Time{} until_param := ctx.Query("until") if until_param != "" { u, err := time.Parse(time.RFC3339, until_param) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } until = u } err := caliopen.Facilities.Notifiers.DeleteNotifications(userId, until) if err != nil { returnedErr := new(swgErr.CompositeError) if err.Code() == DbCaliopenErr && err.Cause().Error() == "not found" { ctx.Status(http.StatusNoContent) return } else { returnedErr = swgErr.CompositeValidationError(err, err.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } ctx.Status(http.StatusNoContent) } // GetNotification handles GET /notifications/:notification_id func GetNotification(ctx *gin.Context) { userID, err := operations.NormalizeUUIDstring(ctx.MustGet("user_id").(string)) notificationID, err := operations.NormalizeUUIDstring(ctx.Param("notification_id")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } notif, e := caliopen.Facilities.Notifiers.RetrieveNotification(userID, notificationID) if e != nil { returnedErr := new(swgErr.CompositeError) if e.Code() == DbCaliopenErr && e.Cause().Error() == "not found" { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), e, e.Cause()) } else { returnedErr = swgErr.CompositeValidationError(e, e.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } else { notif_json, err := notif.MarshalFrontEnd() if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Data(http.StatusOK, "application/json; charset=utf-8", notif_json) } } } // DeleteNotification handles DELETE /notifications/:notification_id func DeleteNotification(ctx *gin.Context) { userID, err := operations.NormalizeUUIDstring(ctx.MustGet("user_id").(string)) notificationID, err := operations.NormalizeUUIDstring(ctx.Param("notification_id")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } e := caliopen.Facilities.Notifiers.DeleteNotification(userID, notificationID) if err != nil { returnedErr := new(swgErr.CompositeError) if e.Code() == DbCaliopenErr && e.Cause().Error() == "not found" { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), e, e.Cause()) } else { returnedErr = swgErr.CompositeValidationError(e, e.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } ctx.Status(http.StatusNoContent) } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/participants/discussion.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package participants import ( "bytes" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "net/http" ) // POST …/participants/discussion // returns canonical hash of participant_uris and if the corresponding discussion_id exists func HashUris(ctx *gin.Context) { userId := ctx.MustGet("user_id").(string) shardId := ctx.MustGet("shard_id").(string) userInfo := &UserInfo{User_id: userId, Shard_id: shardId} var err error var participants []Participant ctx.ShouldBindJSON(&participants) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } hash, _, err := HashFromParticipantsUris(participants) if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } discussion, err := caliopen.Facilities.RESTfacility.DiscussionMetadata(userInfo, hash) if err != nil && err.Error() != "not found" { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } var respBuf bytes.Buffer respBuf.WriteString("{\"discussion_id\":\"" + discussion.DiscussionId + "\",\"hash\":\"" + hash + "\"}") ctx.Data(http.StatusOK, "application/json; charset=utf-8", respBuf.Bytes()) } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/participants/suggest.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package participants import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "net/http" "strings" ) // GET …/participants/suggest?context=xxxx&q=xxx func Suggest(ctx *gin.Context) { user_id := ctx.MustGet("user_id").(string) shard_id := ctx.MustGet("shard_id").(string) user_info := &UserInfo{User_id: user_id, Shard_id: shard_id} query_context := ctx.Request.URL.Query().Get("context") if query_context == "" { e := swgErr.New(http.StatusUnprocessableEntity, "Missing 'context' param in query") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } query_string := ctx.Request.URL.Query().Get("q") if query_string == "" { e := swgErr.New(http.StatusUnprocessableEntity, "Missing 'q' param in query") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } if len(query_string) < 3 { e := swgErr.New(http.StatusUnprocessableEntity, "Query string must be at least 3 chars long.") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } switch query_context { case "msg_compose": // convert string query to lower to benefit the case insensitive search from ES // as suggest is in the context of a msg_compose, ie : we are looking for an address query_string = strings.ToLower(query_string) suggests, err := caliopen.Facilities.RESTfacility.SuggestRecipients(user_info, query_string) if err != nil { e := swgErr.New(http.StatusInternalServerError, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } ctx.JSON(http.StatusOK, suggests) default: e := swgErr.New(http.StatusUnprocessableEntity, "Unknown ") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/providers/oauth-test.html ================================================ Oauth providers test ​
================================================ FILE: src/backend/interfaces/REST/go.server/operations/providers/providers.go ================================================ package providers import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "net/http" ) // GetProvidersList handles get …/providers func GetProvidersList(ctx *gin.Context) { providers, err := caliopen.Facilities.RESTfacility.RetrieveProvidersList() if err != nil && err.Error() != "not found" { e := swgErr.New(http.StatusInternalServerError, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } ret := struct { Total int `json:"total"` Providers []Provider `json:"providers,omitempty"` }{len(providers), providers} ctx.JSON(http.StatusOK, ret) } // GetProvider handles get …/providers/:provider_name func GetProvider(ctx *gin.Context) { userID, err := operations.NormalizeUUIDstring(ctx.GetString("user_id")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } providerName := ctx.Param("provider_name") if providerName == "" { e := swgErr.New(http.StatusUnprocessableEntity, "provider name is empty") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } var identifier string if providerName == "mastodon" { identifier = ctx.Query("identifier") if identifier == "" { // TODO : return registered instances e := swgErr.New(http.StatusUnprocessableEntity, "missing mastodon identifier") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } } provider, errC := caliopen.Facilities.RESTfacility.GetProviderOauthFor(userID, providerName, identifier) if errC != nil { returnedErr := new(swgErr.CompositeError) switch errC.Code() { case NotFoundCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "RESTfacility returned error"), errC, errC.Cause()) case UnprocessableCaliopenErr: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusUnprocessableEntity, "RESTfacility returned error"), errC, errC.Cause()) default: returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "RESTfacility returned error"), errC, errC.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } ctx.JSON(http.StatusOK, provider) } // CallbackHandler handles get …/providers/:provider_name/callback func CallbackHandler(ctx *gin.Context) { provider := ctx.Param("provider_name") switch provider { case "twitter": token := ctx.Query("oauth_token") verifier := ctx.Query("oauth_verifier") _, errC := caliopen.Facilities.RESTfacility.CreateTwitterIdentity(token, verifier) if errC != nil { returnedErr := new(swgErr.CompositeError) returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "RESTfacility returned error"), errC, errC.Cause()) http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } ctx.Status(http.StatusNoContent) case "gmail", "mastodon": state := ctx.Query("state") code := ctx.Query("code") var errC CaliopenError if provider == "gmail" { _, errC = caliopen.Facilities.RESTfacility.CreateGmailIdentity(state, code) } else { if state == "" { ctx.Status(http.StatusNoContent) } _, errC = caliopen.Facilities.RESTfacility.CreateMastodonIdentity(state, code) } if errC != nil { returnedErr := new(swgErr.CompositeError) returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusFailedDependency, "RESTfacility returned error"), errC, errC.Cause()) http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } ctx.Status(http.StatusNoContent) default: e := swgErr.New(http.StatusNotImplemented, "not implemented") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/search.go ================================================ package operations import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" "github.com/satori/go.uuid" "net/http" "strconv" ) func SimpleSearch(ctx *gin.Context) { // temporary hack to check if X-Caliopen-IL header is in request, because go-openapi pkg fails to do it. // (NB : CanonicalHeaderKey func normalize http headers with uppercase at beginning of words) if _, ok := ctx.Request.Header["X-Caliopen-Il"]; !ok { e := swgErr.New(http.StatusFailedDependency, "Missing mandatory header 'X-Caliopen-Il'.") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } user_uuid, _ := uuid.FromString(ctx.MustGet("user_id").(string)) shard_id := ctx.MustGet("shard_id").(string) var user_UUID UUID var limit, offset int user_UUID.UnmarshalBinary(user_uuid.Bytes()) query := ctx.Request.URL.Query() // check request consistency. (see search API readme in doc folder) invalid := false reasons := []error{} if len(query) < 1 || len(query) > 4242 { // why 4242 ? why not ? invalid = true reasons = append(reasons, errors.New("invalid query length")) } term, term_ok := query["term"] if !term_ok { invalid = true reasons = append(reasons, errors.New("'term' param is missing")) } for _, t := range term { if len(t) < 3 { invalid = true reasons = append(reasons, errors.New("'term' param must length 3 chars at least")) } } doc_type, has_doc_type := query["doctype"] if l, ok := query["limit"]; ok { if !has_doc_type { invalid = true reasons = append(reasons, errors.New("'limit' param only allowed if 'doctype' param also provided")) } else { limit, _ = strconv.Atoi(l[0]) } } if o, ok := query["offset"]; ok { if !has_doc_type { invalid = true reasons = append(reasons, errors.New("'offset' param only allowed if 'doctype' param also provided")) } else { offset, _ = strconv.Atoi(o[0]) } } // build the search object search := IndexSearch{ User_id: user_UUID, Shard_id: shard_id, Limit: limit, Offset: offset, ILrange: GetImportanceLevel(ctx), } if field, ok := query["field"]; ok { if len(field) > 1 { invalid = true reasons = append(reasons, errors.New("at most one 'field' param allowed")) } else { search.Terms = map[string][]string{field[0]: query["term"]} // take only first field provided for now } } else { search.Terms = map[string][]string{"_all": query["term"]} } if has_doc_type { if len(doc_type) > 1 { invalid = true reasons = append(reasons, errors.New("at most one 'doctype' param allowed")) } else { switch doc_type[0] { // take only first doctype provided for now case "message": search.DocType = MessageIndexType case "contact": search.DocType = ContactIndexType default: invalid = true reasons = append(reasons, errors.New("'doctype' unknown")) } } } if invalid { e := swgErr.CompositeValidationError(reasons...) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // trigger the search result, err := caliopen.Facilities.RESTfacility.Search(search) // handle response if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { response, err := result.MarshalFrontEnd() if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } ctx.Data(http.StatusOK, "application/json; charset=utf-8", response) } } func AdvancedSearch(ctx *gin.Context) { ctx.AbortWithStatus(http.StatusNotImplemented) } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/tags/tags.go ================================================ package tags import ( "bytes" "errors" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" swgErr "github.com/go-openapi/errors" "github.com/satori/go.uuid" "github.com/tidwall/gjson" "io/ioutil" "net/http" "strconv" "strings" ) // RetrieveUserTags fetches all tags tied to an user, system tags as well as custom tags. func RetrieveUserTags(ctx *gin.Context) { user_id := ctx.MustGet("user_id").(string) tags, err := caliopen.Facilities.RESTfacility.RetrieveUserTags(user_id) if err != nil { returnedErr := new(swgErr.CompositeError) if err.Code() == DbCaliopenErr && err.Cause().Error() == "tags not found" { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), err, err.Cause()) } else { returnedErr = swgErr.CompositeValidationError(err, err.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() } else { var respBuf bytes.Buffer respBuf.WriteString("{\"total\": " + strconv.Itoa(len(tags)) + ",") respBuf.WriteString(("\"tags\":[")) first := true for _, tag := range tags { json_tag, err := tag.MarshalFrontEnd() if err == nil { if first { first = false } else { respBuf.WriteByte(',') } respBuf.Write(json_tag) } } respBuf.WriteString("]}") ctx.Data(http.StatusOK, "application/json; charset=utf-8", respBuf.Bytes()) } } func CreateTag(ctx *gin.Context) { var tag Tag b := binding.JSON if err := b.Bind(ctx.Request, &tag); err == nil { user_uuid, _ := uuid.FromString(ctx.MustGet("user_id").(string)) tag.User_id.UnmarshalBinary(user_uuid.Bytes()) if tag.Label == "" || strings.Replace(tag.Label, " ", "", -1) == "" { err := errors.New("tag's name is empty") e := swgErr.New(http.StatusBadRequest, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } tag.Label = strings.TrimSpace(tag.Label) err := caliopen.Facilities.RESTfacility.CreateTag(&tag) if err != nil { returnedErr := new(swgErr.CompositeError) if err.Code() == DbCaliopenErr && err.Cause().Error() == "not found" { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), err, err.Cause()) } else { returnedErr = swgErr.CompositeValidationError(err, err.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() } else { ctx.JSON(http.StatusOK, struct{ Location string }{ http_middleware.RoutePrefix + http_middleware.TagsRoute + "/" + tag.Name, }) } } else { e := swgErr.New(http.StatusBadRequest, fmt.Sprintf("Unable to json marshal the provided payload : %s", err)) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } } // RetrieveTag fetches a tag with tag_name & user_id func RetrieveTag(ctx *gin.Context) { user_id := ctx.MustGet("user_id").(string) tag_name := ctx.Param("tag_name") if tag_name == "" { e := swgErr.New(http.StatusUnprocessableEntity, "missing tag name") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } if user_id != "" { tag, err := caliopen.Facilities.RESTfacility.RetrieveTag(user_id, tag_name) if err != nil { returnedErr := new(swgErr.CompositeError) if err.Code() == DbCaliopenErr && err.Cause().Error() == "tag not found" { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), err, err.Cause()) } else { returnedErr = swgErr.CompositeValidationError(err, err.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() } else { tag_json, err := tag.MarshalFrontEnd() if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Data(http.StatusOK, "application/json; charset=utf-8", tag_json) } } } else { err := errors.New("invalid params") e := swgErr.New(http.StatusBadRequest, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } } func PatchTag(ctx *gin.Context) { var err error var user_id string var tag_name string if id, ok := ctx.Get("user_id"); !ok { e := swgErr.New(http.StatusUnprocessableEntity, "user_id is missing") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } else { if user_id, err = operations.NormalizeUUIDstring(id.(string)); err != nil { e := swgErr.New(http.StatusUnprocessableEntity, "user_id is invalid") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } } tag_name = ctx.Param("tag_name") if tag_name == "" { e := swgErr.New(http.StatusUnprocessableEntity, "tag_name is missing") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } var patch []byte patch, err = ioutil.ReadAll(ctx.Request.Body) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } e := caliopen.Facilities.RESTfacility.PatchTag(patch, user_id, tag_name) if e != nil { returnedErr := new(swgErr.CompositeError) if e.Code() == DbCaliopenErr && e.Cause().Error() == "tag not found" { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), e, e.Cause()) } else { returnedErr = swgErr.CompositeValidationError(e, e.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } else { ctx.Status(http.StatusNoContent) } } func DeleteTag(ctx *gin.Context) { user_id := ctx.MustGet("user_id").(string) tag_name := ctx.Param("tag_name") if tag_name == "" { e := swgErr.New(http.StatusUnprocessableEntity, "tag_name is missing") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } if user_id != "" { err := caliopen.Facilities.RESTfacility.DeleteTag(user_id, tag_name) if err != nil { returnedErr := new(swgErr.CompositeError) if err.Code() == DbCaliopenErr && err.Cause().Error() == "tag not found" { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), err, err.Cause()) } else { returnedErr = swgErr.CompositeValidationError(err, err.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() } else { ctx.Status(http.StatusNoContent) } } else { err := errors.New("invalid params") e := swgErr.New(http.StatusBadRequest, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } // TODO : remove tag refs. nested in resources } // PatchResourceWithTag apply the payload (a PATCH tag json) to a resource to update its tags func PatchResourceWithTags(ctx *gin.Context) { var err error var userID string var resourceID string var patch []byte var resourceType string if id, ok := ctx.Get("user_id"); !ok { e := swgErr.New(http.StatusBadRequest, "user_id is missing") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } else { if userID, err = operations.NormalizeUUIDstring(id.(string)); err != nil { e := swgErr.New(http.StatusUnprocessableEntity, "user_id is invalid") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } } shard_id, ok := ctx.Get("shard_id") if !ok { e := swgErr.New(http.StatusBadRequest, "shard_id is missing in context") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // parse payload and ensure it is a patch for tags property only patch, err = ioutil.ReadAll(ctx.Request.Body) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } if !gjson.Valid(string(patch)) { e := swgErr.New(http.StatusUnprocessableEntity, "invalid json") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } p := gjson.ParseBytes(patch) p.ForEach(func(key, value gjson.Result) bool { if key.Str != "tags" && key.Str != "current_state" { err = swgErr.New(http.StatusBadRequest, fmt.Sprintf("invalid property <%s> within json", key.Str)) return false } else if key.Str == "current_state" { value.ForEach(func(k, v gjson.Result) bool { if k.Str != "tags" { err = swgErr.New(http.StatusBadRequest, fmt.Sprintf("invalid property <%s> within json", k.Str)) return false } return true }) } return true }) if err != nil { http_middleware.ServeError(ctx.Writer, ctx.Request, err) ctx.Abort() return } // call UpdateResourceWithPatch API with correct resourceType depending on provided param param := ctx.Params[0] switch param.Key { case "contactID": resourceType = ContactType case "message_id": resourceType = MessageType default: err = swgErr.New(http.StatusBadRequest, "missing resource param") if err != nil { http_middleware.ServeError(ctx.Writer, ctx.Request, err) ctx.Abort() return } } if resourceID, err = operations.NormalizeUUIDstring(param.Value); err != nil { err = swgErr.New(http.StatusBadRequest, "resource_id is invalid") } if err != nil { http_middleware.ServeError(ctx.Writer, ctx.Request, err) ctx.Abort() return } user_info := &UserInfo{User_id: userID, Shard_id: shard_id.(string)} e := caliopen.Facilities.RESTfacility.UpdateResourceTags(user_info, resourceID, resourceType, patch) if e != nil { returnedErr := new(swgErr.CompositeError) if e.Code() == DbCaliopenErr && e.Cause().Error() == "not found" { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), e, e.Cause()) } else { returnedErr = swgErr.CompositeValidationError(e, e.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() return } ctx.Status(http.StatusNoContent) } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/users/user.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package users import ( "io/ioutil" "net/http" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/middlewares" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server/operations" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/helpers" "github.com/gin-gonic/gin" swgErr "github.com/go-openapi/errors" ) // PATCH …/users/{user_id} // as of october 2017, PatchUser is partially implemented : // it should be only use to change user's password. func PatchUser(ctx *gin.Context) { var err error auth_user := ctx.MustGet("user_id").(string) user_id, err := operations.NormalizeUUIDstring(ctx.Param("user_id")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // for now, an user can only modify himself if auth_user != user_id { e := swgErr.New(http.StatusUnauthorized, "user can only modify himself") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } body, err := ioutil.ReadAll(ctx.Request.Body) patch, err := helpers.ParsePatch(body) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } err = caliopen.Facilities.RESTfacility.PatchUser(auth_user, patch, caliopen.Facilities.Notifiers) if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Status(http.StatusNoContent) } } // RequestPasswordReset handles an anonymous POST request on /passwords/reset/ with json payload // it will try to trigger a password reset procedure func RequestPasswordReset(ctx *gin.Context) { var payload PasswordResetRequest err := ctx.BindJSON(&payload) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } if payload.RecoveryMail == "" && payload.Username == "" { e := swgErr.New(http.StatusBadRequest, "neither username nor recovery email provided, at least one required") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } err = caliopen.Facilities.RESTfacility.RequestPasswordReset(payload, caliopen.Facilities.Notifiers) if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Status(http.StatusNoContent) } return } // ValidatePassResetToken handles a GET on /passwords/reset/:reset_token // this route does nothing more than responding with a 204 if reset_token is still valid func ValidatePassResetToken(ctx *gin.Context) { token := ctx.Param("reset_token") if token == "" { e := swgErr.New(http.StatusUnprocessableEntity, "reset token is empty") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } _, err := caliopen.Facilities.RESTfacility.ValidatePasswordResetToken(token) if err != nil { e := swgErr.New(http.StatusNotFound, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Status(http.StatusNoContent) } return } // ResetPassword handles POST on /passwords/reset/:reset_token // payload should be a json with new password func ResetPassword(ctx *gin.Context) { token := ctx.Param("reset_token") if token == "" { e := swgErr.New(http.StatusBadRequest, "reset token is empty") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } var payload struct { Password string `json:"password"` } err := ctx.BindJSON(&payload) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, "unable to unmarshal payload : "+err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } err = caliopen.Facilities.RESTfacility.ResetUserPassword(token, payload.Password, caliopen.Facilities.Notifiers) if err != nil { e := swgErr.New(http.StatusFailedDependency, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() } else { ctx.Status(http.StatusNoContent) } } // POST …/users/ func Create(ctx *gin.Context) { e := swgErr.New(http.StatusNotImplemented, "not implemented") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } // POST …/users/{user_id}/actions func Delete(ctx *gin.Context) { var err error var caliopenErr CaliopenError auth_user := ctx.MustGet("user_id").(string) access_token := ctx.MustGet("access_token").(string) user_id, err := operations.NormalizeUUIDstring(ctx.Param("user_id")) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } //for now, an user can only modify himself if auth_user != user_id { e := swgErr.New(http.StatusUnauthorized, "user can only modify himself") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } var payload ActionsPayload err = ctx.BindJSON(&payload) if err != nil { e := swgErr.New(http.StatusUnprocessableEntity, "unable to unmarshal payload : "+err.Error()) http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } password := payload.Params.(map[string]interface{})["password"].(string) if payload.Params == nil || password == "" { e := swgErr.New(http.StatusBadRequest, "Password missing") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } payload.UserId = user_id caliopenErr = caliopen.Facilities.RESTfacility.DeleteUser(ActionsPayload{ Actions: payload.Actions, Params: DeleteUserParams{ Password: password, AccessToken: access_token, }, UserId: payload.UserId, }) if caliopenErr != nil { returnedErr := new(swgErr.CompositeError) if caliopenErr.Code() == DbCaliopenErr && caliopenErr.Cause().Error() == "[CassandraBackend] user not found" { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusNotFound, "db returned not found"), caliopenErr, caliopenErr.Cause()) } else if caliopenErr.Code() == WrongCredentialsErr && caliopenErr.Cause().Error() == "[RESTfacility] DeleteUser Wrong password" { returnedErr = swgErr.CompositeValidationError(swgErr.New(http.StatusUnauthorized, "wrong password"), caliopenErr, caliopenErr.Cause()) } else { returnedErr = swgErr.CompositeValidationError(caliopenErr, caliopenErr.Cause()) } http_middleware.ServeError(ctx.Writer, ctx.Request, returnedErr) ctx.Abort() } else { ctx.Status(http.StatusNoContent) } } func Get(ctx *gin.Context) { e := swgErr.New(http.StatusNotImplemented, "not implemented") http_middleware.ServeError(ctx.Writer, ctx.Request, e) ctx.Abort() return } ================================================ FILE: src/backend/interfaces/REST/go.server/operations/users/username.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package users import ( obj "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main" "github.com/gin-gonic/gin" "net/http" ) // GET …/users/isAvailable func IsAvailable(ctx *gin.Context) { username := ctx.Query("username") if username == "" { ctx.JSON(http.StatusBadRequest, obj.Availability{false, username}) return } available, err := caliopen.Facilities.RESTfacility.UsernameIsAvailable(username) if available && err == nil { ctx.JSON(http.StatusOK, obj.Availability{true, username}) return } ctx.JSON(http.StatusOK, obj.Availability{false, username}) } ================================================ FILE: src/backend/interfaces/REST/go.server/proxy.go ================================================ package rest_api import ( log "github.com/Sirupsen/logrus" "net/http" "net/http/httputil" "net/url" ) func StartProxy(config ProxyConfig) { mux := http.NewServeMux() for path, target := range config.Routes { mux.Handle(path, httputil.NewSingleHostReverseProxy(&url.URL{ Scheme: "http", Host: target, })) } addr := config.Interface + ":" + config.Port log.Printf("HTTP proxy listening on %s", addr) log.Fatal(http.ListenAndServe(addr, mux)) return } type ProxyConfig struct { Interface string `mapstructure:"listen_interface"` ListenPort string `mapstructure:"listen_port"` Port string `mapstructure:"port"` Routes map[string]string `mapstructure:"routes"` } ================================================ FILE: src/backend/interfaces/REST/py.server/CHANGES.rst ================================================ 0.0.1 ----- - Initial version 0.0.2 ----- - routes « threads » renamed to « discussions » ================================================ FILE: src/backend/interfaces/REST/py.server/MANIFEST.in ================================================ include *.txt *.ini *.cfg *.rst ================================================ FILE: src/backend/interfaces/REST/py.server/README.rst ================================================ Entry point =========== This repository is part of CaliOpen platform. For documentation, installation and contribution instructions, please refer to https://caliopen.github.io caliopen.api ============ caliopen_api package is a simple Pyramid container to include CaliOpen Rest API services. Documentation ------------- The complete documentation of the API is available at `/doc/api/swagger.json` (it follows swagger/openAPI specs). Developers can browse and test the API at `http://localhost:6543/api-ui/` Local Installation ------------------ To install local dependencies, use `pip `_: :: pip install -e ".[dev,test]" Then install supported components and include them under caliopen.api.services in pyramid configuration file. :: caliopen.api.services = caliopen.api.user caliopen.api.message You will need storage services running locally, cassandra, elasticsearch and a redis instance. `caliopen.cli `_, should be used to create user and load mail data. Running API ----------- :: cd src/backend/main ./startup Components ---------- Current components are : * `caliopen_api.user` * `caliopen_api.message` * `caliopen_api.base` Tests ----- Tests are launched using `nose `_. :: nosetests -sxv caliopen/api/tests/*.py ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/__init__.py ================================================ # -*- coding: utf-8 -*- __version__ = '0.23.0' import logging from pyramid.config import Configurator from caliopen_storage.config import Configuration log = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG) def main(global_config, **settings): """Caliopen entry point for WSGI application. Load Caliopen configuration and setup a WSGI application with loaded API services. """ # XXX ugly way to init caliopen configuration before pyramid caliopen_config = settings['caliopen.config'].split(':')[1] Configuration.load(caliopen_config, 'global') settings['pyramid_swagger.exclude_paths'] = [r'^/api-ui/?', r'^/doc/api/?', r'^/defs/?'] settings['pyramid_swagger.enable_response_validation'] = True config = Configurator(settings=settings) services = config.registry.settings. \ get('caliopen_api.services', []). \ split('\n') route_prefix = settings.get('caliopen_api.route_prefix') for service in services: log.info('Loading %s service' % service) config.include(service, route_prefix=route_prefix) config.end() return config.make_wsgi_app() ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/base/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .config import includeme DEFAULT_LIMIT = 20 class Api(object): """Base class for all Api.""" def __init__(self, context, request): self.request = request self.context = context def get_limit(self): """Return pagination limit from request else default.""" return int(self.request.params.get('limit', DEFAULT_LIMIT)) def get_offset(self): """Return pagination offset from request else 0.""" return int(self.request.params.get('offset', 0)) ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/base/config.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals import logging from caliopen_storage.helpers.connection import connect_storage from .renderer import TextPlainRenderer, JsonRenderer, PartRenderer from .deserializer import json_deserializer from .exception import ValidationError log = logging.getLogger(__name__) """ Solution from : https://github.com/striglia/pyramid_swagger/issues/177#issuecomment-373220674 """ def swagger_error_view(exc, request): """Format swagger validation error.""" raise ValidationError(exc.child.message) def includeme(config): """Configure REST API.""" connect_storage() config.commit() # configure renderers config.add_renderer('text_plain', TextPlainRenderer) config.add_renderer('json', JsonRenderer) config.add_renderer('simplejson', JsonRenderer) config.add_renderer('part', PartRenderer) # configure specific views for API errors config.scan('caliopen_api.base.errors') config.add_cornice_deserializer('application/json', json_deserializer) swagger_context = 'pyramid_swagger.exceptions.RequestValidationError' config.add_exception_view(context=swagger_context, view=swagger_error_view, renderer='json') config.commit() log.info('Base API configured') ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/base/context.py ================================================ # -*- coding: utf-8 -*- """ The Root Context used when a view did not declare it's own context. """ from __future__ import absolute_import, print_function, unicode_literals from pyramid.security import (Everyone, Authenticated, Allow, NO_PERMISSION_REQUIRED, ALL_PERMISSIONS, ) from pyramid.decorator import reify class DefaultContext(object): """A default request context.""" default_acl = [(Allow, Everyone, NO_PERMISSION_REQUIRED), (Allow, Authenticated, 'authenticated'), ] def __init__(self, request): self.request = request self.return_schema = None self._acl = DefaultContext.default_acl[:] @reify def authenticated_user(self): """ Return the authenticated user. :return: authenticated user :rtype: dict """ user = self.request.authenticated_userid return user @reify def __acl__(self): return self._acl def append_acl(self, role, rights): self._acl.append((Allow, role, rights)) ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/base/deserializer.py ================================================ # -*- coding: utf-8 -*- """Caliopen api deserializers.""" from __future__ import absolute_import, print_function, unicode_literals def json_deserializer(request): """Manage json content type.""" if request.json_body: return request.json_body return request.body ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/base/errors.py ================================================ # -*- coding: utf-8 -*- """ Caliopen API error formatting. See https://caliopen.github.io/rfc/2016/04/18/errors-schema for reference. Output structure for any error from API servier is: { "errors": [ { "description": "string", "type": "string", "values": ["string"], "property": "string", "component": "string", "code": "string" } ] } """ from __future__ import absolute_import, print_function, unicode_literals import logging import sys import traceback import json from pyramid.response import Response from pyramid.view import view_config from pyramid.exceptions import Forbidden from pyramid.httpexceptions import ( HTTPRequestTimeout, HTTPUnauthorized, HTTPInternalServerError, HTTPNotFound, HTTPConflict, HTTPMethodNotAllowed, HTTPServiceUnavailable, HTTPBadRequest, HTTPClientError, HTTPNotImplemented, HTTPUnprocessableEntity, HTTPExpectationFailed, HTTPException, ) from pyramid_swagger.exceptions import RequestValidationError, ResponseValidationError log = logging.getLogger(__name__) def format_response_detail(exc, request): """Format error details for a server error.""" route = (request.matched_route.name if request.matched_route else ('Unknown')) return {'component': route, 'values': exc.code, 'property': exc.__class__.__name__, 'message': exc.message} def format_response(exc, request, details=None): """Format response error, details contains application errors.""" # XXX better design for details as keys are not validated if not details: details = {'values': [exc.code], 'property': None, 'component': 'server'} error = {"errors": [{"description": exc.explanation, "type": exc.title, "values": details['values'], "property": details['property'], "component": details['component'], "message": "{}".format(details['message']), "code": exc.code}]} response = Response(json.dumps(error)) response.content_type = str('application/json; charset=UTF-8') response.status_int = exc.code return response @view_config(context=Forbidden) def http_forbidden(exc, request): """Raise HTTPUnauthorized exception.""" if request.authenticated_userid is None: exc = HTTPUnauthorized(explanation="Invalid credentials") return http_exception(exc, request) @view_config(context=HTTPConflict) @view_config(context=HTTPRequestTimeout) @view_config(context=HTTPInternalServerError) @view_config(context=HTTPNotFound) @view_config(context=HTTPMethodNotAllowed) @view_config(context=HTTPServiceUnavailable) @view_config(context=HTTPClientError) @view_config(context=HTTPNotImplemented) @view_config(context=HTTPExpectationFailed) def http_exception(exc, request): if isinstance(request.exc_info[1], ResponseValidationError): # Formatter swagger validation errors exc = request.exc_info[1] details = format_response_detail(exc, request) return format_response(exc, request, details) @view_config(context=Exception) def internal_server_error(exc, request): if isinstance(exc, HTTPException): if isinstance(exc, Response): return exc else: exc = HTTPInternalServerError() details = format_response_detail(exc, request) response = format_response(exc, request, details) exc_type, exc_value, exc_tb = sys.exc_info() formatted_tb = traceback.format_exception(exc_type, exc_value, exc_tb) log.error('Unexpected error ''{}'': {}'.format(' '.join(formatted_tb), response)) return response @view_config(context=HTTPUnprocessableEntity) @view_config(context=HTTPBadRequest) def http_unprocessable_entity(exc, request): if isinstance(request.exc_info[1], RequestValidationError): # Formatter swagger validation errors exc = request.exc_info[1] details = format_response_detail(exc, request) return format_response(exc, request, details) ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/base/exception.py ================================================ # -*- coding: utf-8 -*- """Specific API exception to be process with Specific view.""" from __future__ import absolute_import, print_function, unicode_literals from pyramid.httpexceptions import HTTPClientError from caliopen_storage.exception import NotFound from caliopen_main.common.errors import (PatchUnprocessable, PatchError, PatchConflict) import logging log = logging.getLogger(__name__) class ValidationError(HTTPClientError): """ subclass of :class:`~HTTPClientError`. This indicates that the body or headers failed validity checks, preventing the server from being able to continue processing. code: 400, title: Bad Request """ code = 400 title = 'Bad Request' explanation = ('The server could not comply with the request since ' 'it is either malformed or otherwise incorrect.') class AuthenticationError(HTTPClientError): """ subclass of :class:`~HTTPClientError`. This indicates that the user authentication failed. code: 401, title: Unauthorized """ code = 401 title = 'Authentication error' explanation = 'Wrong credentials (e.g., bad password or username)' class NotAcceptable(HTTPClientError): """ subclass of :class:`~HTTPClientError`. This indicates that the body or headers failed validity checks, preventing the server from being able to continue processing. code: 406, title: Not acceptable """ code = 406 title = 'Not acceptable' explanation = 'Server cannot fulfill the request with given payload' class ResourceNotFound(HTTPClientError): """ subclass of :class:`~HTTPClientError`. This indicates that the body or headers failed validity checks, preventing the server from being able to continue processing. code: 404, title: Not found """ code = 404 title = 'Not Found' explanation = 'The resource could not be found.' class MethodNotAllowed(HTTPClientError): """ subclass of :class:`~HTTPClientError`. This indicates that the body or headers failed validity checks, preventing the server from being able to continue processing. code: 405, title: Method not allowed """ code = 405 title = 'Method not allowed' explanation = 'The method is not allowed or not yet implemented' class MergePatchError(HTTPClientError): """Merge error during patch method.""" def __init__(self, error=None): self.message = error.message if isinstance(error, NotFound): self.code = 404 self.title = "Not Found" self.explanation = "The resource could not be found to apply PATCH" elif isinstance(error, PatchUnprocessable): self.code = 422 self.title = "Patch Unprocessable" self.explanation = "PATCH payload was malformed or unprocessable" elif isinstance(error, PatchError): self.code = 422 self.title = "Patch Error" self.explanation = "Application encountered an error when " \ "applying patch" elif isinstance(error, PatchConflict): self.code = 409 self.title = "Patch Conflict" self.explanation = "The request cannot be applied given " \ "the state of the resource" class Unprocessable(HTTPClientError): """ subclass of :class:`~HTTPClientError`. This indicates that the body or headers failed validity checks, preventing the server from being able to continue processing. code: 422, title: Bad Request """ code = 422 title = 'Unprocessable entity' explanation = 'The method is not allowed or not yet implemented' class MethodFailure(HTTPClientError): """ subclass of :class:`~HTTPClientError`. This indicates that the body or headers failed validity checks, preventing the server from being able to continue processing. code: 424, title: Method failure """ code = 424 title = 'Method failure' explanation = 'The method failed for a dependency reason' ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/base/renderer.py ================================================ # -*- coding: utf-8 -*- """Caliopen pyramid renderer.""" from __future__ import absolute_import, print_function, unicode_literals import logging import datetime import pytz from uuid import UUID from decimal import Decimal import simplejson as json from zope.interface import implementer from pyramid.interfaces import ITemplateRenderer log = logging.getLogger(__name__) @implementer(ITemplateRenderer) class TextPlainRenderer(object): def __init__(self, request): self.request = request def __call__(self, value, system): request = system['request'] request.response.content_type = b'text/plain' return value class JSONEncoder(json.JSONEncoder): def default(self, obj): '''Convert object to JSON encodable type.''' if isinstance(obj, Decimal): return float(obj) if isinstance(obj, datetime.datetime): if obj.tzinfo is None: return str(obj.replace(tzinfo=pytz.utc)) if isinstance(obj, datetime.date): return obj.isoformat() if isinstance(obj, UUID): return str(obj) return super(JSONEncoder, self).default(obj) @implementer(ITemplateRenderer) class JsonRenderer(object): """ Template Factory for render json that accept datetime and decimal. """ def __init__(self, _renderer_helper): pass def __call__(self, data, context): acceptable = ('application/json', 'text/json', 'text/plain') response = context['request'].response content_type = (context['request'].accept.best_match(acceptable) or acceptable[0]) response.content_type = str(content_type) return json.dumps(data, cls=JSONEncoder) @implementer(ITemplateRenderer) class PartRenderer(object): """ Renderer for a message part, content type is defined in the part """ def __init__(self, request): self.request = request def __call__(self, part, context): response = context['request'].response response.content_type = part['part'].content_type.encode('utf-8') return part['part'].payload.encode('utf-8') ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/discussion/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .config import includeme ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/discussion/config.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals import logging log = logging.getLogger(__name__) def includeme(config): """ Serve discussion related REST API. """ config.commit() log.debug('Loading participants discussion API') config.scan('caliopen_api.discussion.participants') ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/discussion/participants.py ================================================ import logging from cornice.resource import resource, view from ..base import Api from caliopen_main.participant.core import hash_participants_uri from caliopen_main.participant.objects import Participant log = logging.getLogger(__name__) @resource(collection_path='/participants/discussion', path='/participants/discussion') class ParticipantDiscussion(Api): """ returns canonical hash of participant_uris which is the corresponding discussion_id """ def __init__(self, request): self.request = request self.user = request.authenticated_userid @view(renderer='json', permission='authenticated') def collection_post(self): parts = self.request.swagger_data['participants'] participants = [] for part in parts: participant = Participant() participant.address = part['address'] participant.label = part['label'] participant.protocol = part['protocol'] participant.contact_id = part.get('contact_ids', []) participants.append(participant) uris = hash_participants_uri(participants) return {'hash': uris['hash'], 'discussion_id': uris['hash']} ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/message/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .config import includeme ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/message/config.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals import logging log = logging.getLogger(__name__) def includeme(config): """ Serve message and discussion REST API. """ config.commit() # Activate cornice in any case and scan log.debug('Loading message API') config.scan('caliopen_api.message.message') ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/message/message.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals import logging from cornice.resource import resource, view from pyramid.response import Response from caliopen_main.message.objects.message import Message as ObjectMessage from caliopen_main.message.core import RawMessage from caliopen_storage.exception import NotFound from ..base import Api from ..base.exception import (ResourceNotFound, MergePatchError) from pyramid.httpexceptions import HTTPServerError, HTTPMovedPermanently from caliopen_pi.features import marshal_features log = logging.getLogger(__name__) @resource(collection_path='/messages', path='/messages/{message_id}') class Message(Api): def __init__(self, request): self.request = request self.user = request.authenticated_userid @view(renderer='json', permission='authenticated') def collection_post(self): data = self.request.json if 'privacy_features' in data: features = marshal_features(data['privacy_features']) data['privacy_features'] = features # ^ json payload should have been validated by swagger module try: message = ObjectMessage.create_draft(user=self.user, **data) except Exception as exc: log.exception(exc) raise MergePatchError(error=exc) message_url = self.request.route_path('message', message_id=str( message.message_id)) message_url = message_url.replace("/v1/", "/v2/") self.request.response.location = message_url.encode('utf-8') return {'location': message_url} @view(renderer='json', permission='authenticated') def patch(self): """Update a message with payload. method follows the rfc5789 PATCH and rfc7396 Merge patch specifications, + 'current_state' caliopen's specs. stored messages are modified according to the fields within the payload, ie payload fields squash existing db fields, no other modification done. If message doesn't existing, response is 404. If payload fields are not conform to the message db schema, response is 422 (Unprocessable Entity). Successful response is 204, without a body. """ message_id = self.request.swagger_data["message_id"] patch = self.request.json if 'privacy_features' in patch: features = marshal_features(patch['privacy_features']) patch['privacy_features'] = features if 'privacy_features' in patch.get('current_state', {}): current = patch['current_state']['privacy_features'] features = marshal_features(current) patch['current_state']['privacy_features'] = features message = ObjectMessage(user=self.user, message_id=message_id) try: message.patch_draft(self.user, patch, db=True, index=True, with_validation=True) except Exception as exc: raise MergePatchError(exc) return Response(None, 204) @view(renderer='json', permission='authenticated') def delete(self): message_id = self.request.swagger_data["message_id"] message = ObjectMessage(user=self.user, message_id=message_id) try: message.get_db() message.get_index() except NotFound: raise ResourceNotFound try: message.delete_db() message.delete_index() except Exception as exc: raise HTTPServerError(exc) return Response(None, 204) @resource(path='/raws/{raw_msg_id}') class Raw(Api): """returns a raw message""" def __init__(self, request): self.request = request self.user = request.authenticated_userid @view(renderer='text_plain', permission='authenticated') def get(self): # XXX how to check privacy_index ? raw_msg_id = self.request.matchdict.get('raw_msg_id') raw = RawMessage.get_for_user(self.user.user_id, raw_msg_id) if raw: return raw.raw_data raise ResourceNotFound('No such message') ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/user/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .config import includeme ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/user/authentication.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals import logging import base64 import ecdsa import hashlib from asn1crypto.core import Sequence, Integer from zope.interface import implements, implementer from pyramid.interfaces import IAuthenticationPolicy, IAuthorizationPolicy from pyramid.security import Everyone, NO_PERMISSION_REQUIRED from caliopen_main.user.core import User from ..base.exception import AuthenticationError log = logging.getLogger(__name__) class EcdsaSignature(Sequence): """Asn.1 structure for an ECDSA signature.""" _fields = [ ('r', Integer), ('s', Integer)] class AuthenticatedUser(object): """Represent an authenticated user.""" def __init__(self, request): self.request = request self.user_id = None self.device_id = None self.shard_id = None self._check_user() # self._load_user() def _check_user(self): if 'Authorization' not in self.request.headers: raise AuthenticationError authorization = self.request.headers['Authorization'].split() if authorization[0] != 'Bearer' and len(authorization) != 2: raise AuthenticationError log.debug('Authentication via Access Token') auth = base64.decodestring(authorization[1]) # authentication values is user_id:token if ':' not in auth: raise AuthenticationError user_id, token = auth.split(':') device_id = None device_header = self.request.headers.get('X-Caliopen-Device-ID', None) if device_header: device_id = device_header cache_key = '{}-{}'.format(user_id, device_id) else: raise AuthenticationError infos = self.request.cache.get(cache_key) if not infos: raise AuthenticationError if infos.get('access_token') != token: raise AuthenticationError if infos.get('user_status', 'unknown') in ['locked', 'maintenance']: raise AuthenticationError('Status {} does not permit operations'. format(infos.get('user_status'))) if self.request.headers.get('X-Caliopen-Device-Signature', None): valid = self._validate_signature(self.request, device_id, infos) log.info('Signature verification for device %s: %r' % (device_id, valid)) self.user_id = user_id self.device_id = device_id self.shard_id = infos['shard_id'] self.access_token = token self._user = None def _get_il_range(self): il_range = self.request.headers.get('X-Caliopen-IL', None) if not il_range: log.warn('No X-Caliopen-IL header') raise ValueError min_il, max_il = il_range.split(';', 1) try: return (int(min_il), int(max_il)) except ValueError: log.error('Invalid value for IL {}'.format(il_range)) raise ValueError except Exception as exc: log.error('Invalid range for IL {}: {}'.format(il_range, exc)) raise exc def _validate_signature(self, request, device_id, infos): """Validate device signature.""" if infos['curve'] == 'P-256': curve = ecdsa.ecdsa.curve_256 crv = ecdsa.curves.NIST256p hashfunc = hashlib.sha256 else: log.warn('Unsupported curve %r' % infos['curve']) return False try: point = ecdsa.ellipticcurve.Point(curve, infos['x'], infos['y']) except AssertionError: log.warn('Invalid curve points') return False sign_header = request.headers.get('X-Caliopen-Device-Signature', None) if sign_header: data = '{}{}{}'.format(request.method, request.path_qs, '') try: b64_header = base64.decodestring(sign_header) ecdsasign = EcdsaSignature.load(b64_header) signature = ecdsasign['r'].contents + ecdsasign['s'].contents vk = ecdsa.VerifyingKey.from_public_point(point, crv, hashfunc=hashfunc) return vk.verify(signature, data) except ecdsa.BadSignatureError: pass except Exception as exc: log.error('Exception during signature verification %r' % exc) return False else: log.warn('no device signature') return False def _load_user(self): if self._user: return self._user = User.get(self.user_id) @property def id(self): self._load_user() return self._user.user_id @property def username(self): self._load_user() return self._user.name @property def contact(self): self._load_user() return self._user.contact class AuthenticationPolicy(object): """Global authentication policy.""" implements(IAuthenticationPolicy) def authenticated_userid(self, request): if hasattr(request, '_CaliopenUser'): return request._CaliopenUser try: request._CaliopenUser = AuthenticatedUser(request) except AuthenticationError: return None return request._CaliopenUser def effective_principals(self, request): account = self.authenticated_userid(request) if not account: return [Everyone] return ["%s:%s" % (account.user_id, account.access_token)] def unauthenticated_userid(self, request): try: return AuthenticatedUser(request) except AuthenticationError: return None def remember(self, request, principal, **kw): """Token Key mechanism can't remember anyone.""" return [] def forget(self, request): return [('WWW-Authenticate', 'Bearer realm="Caliopen"')] @implementer(IAuthorizationPolicy) class AuthorizationPolicy(object): """Basic authorization policy.""" def permits(self, context, principals, permission): """ Return an instance of :class:`pyramid.security.ACLAllowed` instance if the policy permits access, return an instance of :class:`pyramid.security.ACLDenied` if not.""" if permission == NO_PERMISSION_REQUIRED: return True if not principals: False token = principals[0] if ':' in token and permission == 'authenticated': # All managed objects belong to authenticated user # no other policy to apply return True return False def principals_allowed_by_permission(self, context, permission): raise NotImplementedError ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/user/config.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals import logging from .authentication import AuthenticationPolicy, AuthorizationPolicy log = logging.getLogger(__name__) def includeme(config): """Configure REST API for user and contact.""" config.set_authentication_policy(AuthenticationPolicy()) config.set_authorization_policy(AuthorizationPolicy()) log.debug('Loading user API') config.scan('caliopen_api.user.user') log.debug('Loading contact API') config.scan('caliopen_api.user.contact') log.debug('Loading imports API') config.scan('caliopen_api.user.imports') log.debug('Loading settings API') config.scan('caliopen_api.user.settings') ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/user/contact.py ================================================ # -*- coding: utf-8 -*- """Caliopen Contact REST API.""" from __future__ import absolute_import, print_function, unicode_literals import logging import uuid from cornice.resource import resource, view from pyramid.response import Response from pyramid.httpexceptions import HTTPServerError, HTTPForbidden from caliopen_main.common.errors import ForbiddenAction from caliopen_main.contact.core import Contact as CoreContact from caliopen_main.contact.objects.contact import Contact as ContactObject from caliopen_main.contact.returns import ReturnContact from caliopen_main.contact.parameters import NewContact as NewContactParam from ..base import Api from ..base.exception import (ResourceNotFound, ValidationError, MergePatchError) log = logging.getLogger(__name__) @resource(collection_path='/contacts', path='/contacts/{contact_id}') class Contact(Api): """Contact API.""" def __init__(self, request): self.request = request self.user = request.authenticated_userid @view(renderer='json', permission='authenticated') def collection_get(self): filter_params = {'limit': self.get_limit(), 'offset': self.get_offset()} log.debug('Filter parameters {}'.format(filter_params)) results = CoreContact._model_class.search(self.user, **filter_params) data = [] for item in results: try: c = ReturnContact.build( CoreContact.get(self.user, item.contact_id)). \ serialize() data.append(c) except Exception as exc: log.error("unable to serialize contact : {}".format(exc)) return {'contacts': data, 'total': results.hits.total} @view(renderer='json', permission='authenticated') def get(self): contact_id = self.request.swagger_data["contact_id"] try: uuid.UUID(contact_id) except Exception as exc: log.error("unable to extract contact_id: {}".format(exc)) raise ValidationError(exc) contact = ContactObject(user=self.user, contact_id=contact_id) try: contact.get_db() contact.unmarshall_db() except Exception as exc: log.warn(exc) raise ResourceNotFound(detail=exc.message) return contact.marshall_json_dict() @view(renderer='json', permission='authenticated') def collection_post(self): """Create a new contact from json post data structure.""" data = self.request.json contact_param = NewContactParam(data) try: contact_param.validate() if hasattr(contact_param, "tags") and contact_param.tags: raise ValidationError( "adding tags through parent object is forbidden") except Exception as exc: raise ValidationError(exc) contact = CoreContact.create(self.user, contact_param) contact_url = self.request.route_path('contact', contact_id=contact.contact_id) self.request.response.location = contact_url.encode('utf-8') # XXX return a Location to get contact not send it direct return {'location': contact_url} @view(renderer='json', permission='authenticated') def patch(self): """Update a contact with payload. method follows the rfc5789 PATCH and rfc7396 Merge patch specifications, + 'current_state' caliopen's specs. stored messages are modified according to the fields within the payload, ie payload fields squash existing db fields, no other modification done. If message doesn't existing, response is 404. If payload fields are not conform to the message db schema, response is 422 (Unprocessable Entity). Successful response is 204, without a body. """ contact_id = self.request.swagger_data["contact_id"] patch = self.request.json contact = ContactObject(user=self.user, contact_id=contact_id) try: contact.apply_patch(patch, db=True, index=True, with_validation=True) except Exception as exc: raise MergePatchError(error=exc) return Response(None, 204) @view(renderer='json', permission='authenticated') def delete(self): contact_id = self.request.swagger_data["contact_id"] contact = ContactObject(user=self.user, contact_id=contact_id) try: contact.delete() except Exception as exc: if isinstance(exc, ForbiddenAction): raise HTTPForbidden(exc) else: raise HTTPServerError(exc) return Response(None, 204) ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/user/imports.py ================================================ # -*- coding: utf-8 -*- """Caliopen file import REST API.""" from __future__ import absolute_import, print_function, unicode_literals import logging from caliopen_main.contact.core import Contact as CoreContact from cornice.resource import resource, view from pyramid.response import Response from caliopen_main.contact.parsers import VcardParser from ..base.exception import (ValidationError, Unprocessable) from ..base import Api log = logging.getLogger(__name__) @resource(collection_path='/imports', path='') class ContactImport(Api): def __init__(self, request): self.request = request self.user = request.authenticated_userid @view(permission='authenticated') def collection_post(self): """API to import an user file (vcard at this time).""" # need to check by ourself if param is present # because swagger lib failed to do it correctly :( try: self.request.POST.getone("file") except Exception as exc: raise ValidationError(exc) data = self.request.POST['file'].file try: parser = VcardParser(data) except Exception as exc: log.exception('Exception during vcard file parsing %r' % exc) raise ValidationError(exc) try: new_contacts = parser.parse() except Exception as exc: log.error('Syntax error: {}'.format(exc)) raise ValidationError(exc) try: for contact in new_contacts: CoreContact.create(self.user, contact.contact) except Exception as exc: log.error( 'File valid but we can create the new contact: {}'.format(exc)) raise Unprocessable(detail=exc.message) return Response(status=200) ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/user/settings.py ================================================ # -*- coding: utf-8 -*- """Caliopen ReST API for user settings management.""" from __future__ import absolute_import, print_function, unicode_literals import logging from pyramid.response import Response from pyramid.httpexceptions import HTTPNoContent from caliopen_main.user.objects.settings import Settings as ObjectSettings from cornice.resource import resource, view from ..base import Api from ..base.context import DefaultContext from ..base.exception import MergePatchError log = logging.getLogger(__name__) @resource(path='/settings', name='Settings', factory=DefaultContext ) class SettingsAPI(Api): """Settings management API.""" def __init__(self, request, context): """Create an instance of Device API.""" self.request = request self.user = request.authenticated_userid @view(renderer='json', permission='authenticated') def get(self): """Return user settings.""" objects = ObjectSettings.list_db(self.user) settings = [x.marshall_dict() for x in objects] if settings: return settings[0] raise HTTPNoContent() @view(renderer='json', permission='authenticated') def patch(self): """Update settings with payload. method follows the rfc5789 PATCH and rfc7396 Merge patch specifications, + 'current_state' caliopen's specs. stored messages are modified according to the fields within the payload, ie payload fields squash existing db fields, no other modification done. If message doesn't existing, response is 404. If payload fields are not conform to the message db schema, response is 422 (Unprocessable Entity). Successful response is 204, without a body. """ patch = self.request.json settings = ObjectSettings(self.user) error = settings.apply_patch(patch, db=True) if error is not None: raise MergePatchError(error) return Response(None, 204) ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/user/user.py ================================================ # -*- coding: utf-8 -*- """Caliopen user API.""" from __future__ import absolute_import, print_function, unicode_literals import logging import datetime from pyramid.security import NO_PERMISSION_REQUIRED from cornice.resource import resource, view import tornado.ioloop import tornado.gen from nats.io import Client as Nats import json from ..base.context import DefaultContext from .util import create_token from ..base import Api from ..base.exception import AuthenticationError, NotAcceptable from ..base.exception import Unprocessable, ValidationError from caliopen_storage.exception import NotFound from caliopen_storage.config import Configuration from caliopen_main.common.core import PublicKey from caliopen_main.user.core import User from caliopen_main.user.parameters import NewUser, Settings from caliopen_main.user.returns.user import ReturnUser from caliopen_main.contact.parameters import NewContact, NewEmail from caliopen_main.device.core import Device log = logging.getLogger(__name__) def get_device_sig_key(user, device): """Get device signature key.""" keys = PublicKey._model_class.filter(user_id=user.user_id, resource_id=device.device_id) keys = [x for x in keys if x.resource_type == 'device' and x.use == 'sig'] if keys: return keys[0] return None def patch_device_key(key, param): """Patch a device signature public key as X and Y points are not valid.""" if not key.x and not key.y: key.x = int(param['ecdsa_key']['x'], 16) key.y = int(param['ecdsa_key']['y'], 16) key.save() return True return False def make_user_device_tokens(request, user, device, key, ttl=86400): """Return (key, tokens) informations for cache entry management.""" cache_key = '{}-{}'.format(user.user_id, device.device_id) previous = request.cache.get(cache_key) if previous: status = previous.get('user_status', 'unknown') log.info('Found current user device entry {} : {}'. format(cache_key, status)) if status in ['locked', 'maintenance']: raise AuthenticationError('Status {} does not permit operations'. format(status)) access_token = create_token() refresh_token = create_token(80) expires_at = (datetime.datetime.utcnow() + datetime.timedelta(seconds=ttl)) tokens = {'access_token': access_token, 'refresh_token': refresh_token, 'expires_in': ttl, # TODO : remove this value 'shard_id': user.shard_id, 'expires_at': expires_at.isoformat(), 'user_status': 'active', 'key_id': str(key.key_id), 'x': key.x, 'y': key.y, 'curve': key.crv} request.cache.set(cache_key, tokens) result = tokens.copy() result.pop('shard_id') return result @resource(path='', collection_path='/authentications', name='Authentication', factory=DefaultContext ) class AuthenticationAPI(Api): """User authentication API.""" @view(renderer='json', permission=NO_PERMISSION_REQUIRED) def collection_post(self): """ Api for user authentication. Store generated tokens in a cache entry related to user_id and return a structure with this tokens for client usage. """ params = self.request.json try: user = User.authenticate(params['username'], params['password']) log.info('Authenticate user {username}'.format(username=user.name)) except Exception as exc: log.info('Authentication error for {name} : {error}'. format(name=params['username'], error=exc)) raise AuthenticationError(detail=exc.message) # Device management in_device = self.request.swagger_data['authentication']['device'] key = None if in_device: try: device = Device.get(user, in_device['device_id']) log.info("Found device %s" % device.device_id) # Found a device, check if signature public key have X and Y key = get_device_sig_key(user, device) if not key: log.error('No signature key found for device %r' % device.device_id) else: if patch_device_key(key, in_device): log.info('Patch device key OK') else: log.warn('Patch device key does not work') except NotFound: devices = Device.find(user) if devices.get('objects', []): in_device['status'] = 'unverified' else: in_device['name'] = 'default' # we must declare a new device device = Device.create_from_parameter(user, in_device, self.request.headers) log.info("Created device %s" % device.device_id) key = get_device_sig_key(user, device) if not key: log.error('No signature key found for device %r' % device.device_id) else: raise ValidationError(detail='No device informations') try: device.login(self.request.headers.get('X-Forwarded-For')) except Exception as exc: log.exception('Device login failed: {0}'.format(exc)) tokens = make_user_device_tokens(self.request, user, device, key) return {'user_id': user.user_id, 'username': user.name, 'tokens': tokens, 'device': {'device_id': device.device_id, 'status': device.status}} def no_such_user(request): """Validator that an user does not exist.""" username = request.swagger_data['user']['username'] if not User.is_username_available(username): raise NotAcceptable(detail='User already exist') @resource(path='/users/{user_id}', collection_path='/users', name='User', factory=DefaultContext) class UserAPI(Api): """User API.""" @view(renderer='json', permission=NO_PERMISSION_REQUIRED, validators=no_such_user) def collection_post(self): """Create a new user.""" settings = Settings() settings.import_data(self.request.swagger_data['user']['settings']) try: settings.validate() except Exception as exc: raise Unprocessable(detail=exc.message) param = NewUser({'name': self.request.swagger_data['user']['username'], 'password': self.request.swagger_data['user'][ 'password'], 'recovery_email': self.request.swagger_data['user'][ 'recovery_email'], 'settings': settings, }) if self.request.swagger_data['user']['contact'] is not None: param.contact = self.request.swagger_data['user']['contact'] else: c = NewContact() c.given_name = param.name c.family_name = "" # can't guess it ! email = NewEmail() email.address = param.recovery_email c.emails = [email] param.contact = c try: user = User.create(param) except Exception as exc: log.exception('Error during user creation {0}'.format(exc)) raise NotAcceptable(detail=exc.message) log.info('Created user {} with name {}'. format(user.user_id, user.name)) # default device management in_device = self.request.swagger_data['user']['device'] if in_device: try: in_device['name'] = 'default' device = Device.create_from_parameter(user, in_device, self.request.headers) log.info('Device %r created' % device.device_id) except Exception as exc: log.exception('Error during default device creation %r' % exc) else: log.warn('Missing default device parameter') user_url = self.request.route_path('User', user_id=user.user_id) self.request.response.location = user_url.encode('utf-8') # send notification to apiv2 to trigger post-registration actions config = Configuration('global').get("message_queue") try: tornado.ioloop.IOLoop.current().run_sync( lambda: notify_new_user(user, config), timeout=5) except Exception as exc: log.exception( 'Error when sending new_user notification on NATS : {0}'. format(exc)) return {'location': user_url} @resource(path='/me', name='MeUser', factory=DefaultContext) class MeUserAPI(Api): """Me API.""" @view(renderer='json', permission='authenticated') def get(self): """Get information about logged user.""" user_id = self.request.authenticated_userid.user_id user = User.get(user_id) return ReturnUser.build(user).serialize() @tornado.gen.coroutine def notify_new_user(user, config): client = Nats() server = 'nats://{}:{}'.format(config['host'], config['port']) opts = {"servers": [server]} yield client.connect(**opts) notif = { 'message': 'created', 'user_name': '{0}'.format(user.name), 'user_id': '{0}'.format(user.user_id) } yield client.publish('userAction', json.dumps(notif)) yield client.flush() log.info("New user notification sent on NATS for {0}".format(user.user_id)) ================================================ FILE: src/backend/interfaces/REST/py.server/caliopen_api/user/util.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals import os import binascii def create_token(size=40): return binascii.hexlify(os.urandom(int(size/2))).decode('ascii') ================================================ FILE: src/backend/interfaces/REST/py.server/requirements.deps ================================================ caliopen_storage caliopen_main ================================================ FILE: src/backend/interfaces/REST/py.server/setup.cfg ================================================ [nosetests] match = ^test nocapture = 1 cover-package = caliop with-coverage = 1 cover-erase = 1 [compile_catalog] directory = caliop/locale domain = caliop statistics = true [extract_messages] add_comments = TRANSLATORS: output_file = caliop/locale/caliop.pot width = 80 [init_catalog] domain = caliop input_file = caliop/locale/caliop.pot output_dir = caliop/locale [update_catalog] domain = caliop input_file = caliop/locale/caliop.pot output_dir = caliop/locale previous = true ================================================ FILE: src/backend/interfaces/REST/py.server/setup.py ================================================ import os import sys import re from setuptools import setup, find_packages PY3 = sys.version_info[0] == 3 name = "caliopen_api" here = os.path.abspath(os.path.dirname(__file__)) README = open(os.path.join(here, 'README.rst')).read() CHANGES = open(os.path.join(here, 'CHANGES.rst')).read() with open( os.path.join(*([here] + name.split('.') + ['__init__.py']))) as v_file: version = re.compile(r".*__version__ = '(.*?)'", re.S).match( v_file.read()).group(1) requires = [ 'pyramid', 'pyramid_jinja2', 'redis==2.10.6', # Enforce this version, version >= 3.0.0 break setex 'pyramid_kvs==0.3.0', 'waitress', 'cornice==1.2.1', 'colander', 'pyramid-swagger', 'rfc3987', 'webcolors', 'strict-rfc3339', 'nats-client', 'tornado==4.2', 'gunicorn', 'ecdsa'] if (os.path.isfile('./requirements.deps')): with open('./requirements.deps') as f_deps: requires.extend(f_deps.read().split('\n')) tests_require = ['nose', 'coverage'] if sys.version_info < (3, 3): tests_require.append('mock') extras_require = { 'dev': [ 'pyramid_debugtoolbar', 'caliopen_api_doc', ], 'doc': [ 'sphinx', ], 'test': tests_require } setup(name=name, version=version, description='Caliopen REST API Server', long_description=README + '\n\n' + CHANGES, classifiers=[ "Programming Language :: Python", "Framework :: Pyramid", "Topic :: Internet :: WWW/HTTP", "Topic :: Internet :: WWW/HTTP :: WSGI :: Application", ], author='Caliopen Contributors', author_email='', url='https://github.com/Caliopen/caliopen.api', license='AGPLv3', keywords='web pyramid api rest', packages=find_packages(), include_package_data=True, zip_safe=False, install_requires=requires, tests_require=tests_require, extras_require=extras_require, test_suite="caliopen_api.tests", entry_points={ 'paste.app_factory': ['main = caliopen_api:main'], }) ================================================ FILE: src/backend/main/go.backends/AttachmentsInterfaces.go ================================================ package backends import ( "io" ) type AttachmentStorage interface { StoreAttachment(attachment_id string, file io.Reader) (uri string, size int, err error) GetAttachment(uri string) (file io.Reader, err error) DeleteAttachment(uri string) error } ================================================ FILE: src/backend/main/go.backends/CacheInterfaces.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backends import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "time" ) type APICache interface { // authentication GetAuthToken(token string) (value *Auth_cache, err error) LogoutUser(key string) error // password reset process GetResetPasswordToken(token string) (*TokenSession, error) GetResetPasswordSession(user_id string) (*TokenSession, error) SetResetPasswordSession(user_id, reset_token string) (*TokenSession, error) DeleteResetPasswordSession(user_id string) error // Oauth session handling SetOauthSession(key string, session *OauthSession) error GetOauthSession(key string) (*OauthSession, error) DeleteOauthSession(user_id string) error // Device validation GetDeviceValidationSession(userId, deviceId string) (*TokenSession, error) GetTokenValidationSession(userId, token string) (*TokenSession, error) SetDeviceValidationSession(userId, deviceId, token string) (*TokenSession, error) DeleteDeviceValidationSession(userId, deviceId string) error } type CacheBackend interface { // CRD interface to the underlying backend Set(key string, value []byte, ttl time.Duration) error Get(key string) (value []byte, err error) Del(key string) error } ================================================ FILE: src/backend/main/go.backends/ContactsInterfaces.go ================================================ /* * // Copyleft (ɔ) 2017 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package backends import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type ContactStorage interface { CreateContact(contact *Contact) error RetrieveContact(userID, contactID string) (contact *Contact, err error) RetrieveUserContactId(userID string) string UpdateContact(contact, oldContact *Contact, fields map[string]interface{}) error DeleteContact(contact *Contact) error ContactExists(userId, contactId string) bool LookupContactsByIdentifier(user_id, address, kind string) (contact_ids []string, err error) ContactsForParticipants(userID string, participants map[string]Participant) error } type ContactIndex interface { CreateContact(user *UserInfo, contact *Contact) error DeleteContact(user *UserInfo, contact *Contact) error UpdateContact(user *UserInfo, contact *Contact, fields map[string]interface{}) error FilterContacts(search IndexSearch) (Contacts []*Contact, totalFound int64, err error) } ================================================ FILE: src/backend/main/go.backends/CredentialsInterfaces.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package backends import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type CredentialsStorage interface { CreateCredentials(userIdentity *UserIdentity, cred Credentials) error RetrieveCredentials(userId, remoteId string) (Credentials, error) UpdateCredentials(userId, remoteId string, cred Credentials) error DeleteCredentials(userId, remoteId string) error } ================================================ FILE: src/backend/main/go.backends/DevicesInterfaces.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package backends import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type DevicesStorage interface { CreateDevice(device *Device) error RetrieveDevices(user_id string) (devices []Device, err error) RetrieveDevice(userId, deviceId string) (device *Device, err error) UpdateDevice(device, oldDevice *Device, modifiedFields map[string]interface{}) error DeleteDevice(device *Device) error } ================================================ FILE: src/backend/main/go.backends/DiscussionsInterface.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package backends import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type DiscussionStorage interface { GetUserLookupHashes(userId UUID, kind, key string) (hashes []ParticipantHash, err error) UpsertDiscussionLookups(userId UUID, participants []Participant) error } type DiscussionIndex interface { GetDiscussionsList(filter IndexSearch, withIL bool) ([]Discussion, error) } ================================================ FILE: src/backend/main/go.backends/IdentitiesInterface.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package backends import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type ( IdentityStorage interface { RetrieveLocalsIdentities(user_id string) ([]UserIdentity, error) CreateUserIdentity(userIdentity *UserIdentity) CaliopenError RetrieveUserIdentity(userId, RemoteId string, withCredentials bool) (*UserIdentity, error) LookupIdentityByIdentifier(string, ...string) ([][2]string, error) LookupIdentityByType(string, ...string) ([][2]string, error) IdentityStorageUpdater DeleteUserIdentity(userIdentity *UserIdentity) error RetrieveRemoteIdentities(userId string, withCredentials bool) ([]*UserIdentity, error) RetrieveAllRemotes(withCredentials bool) (<-chan *UserIdentity, error) UpdateRemoteInfosMap(userId, remoteId string, infos map[string]string) error RetrieveRemoteInfosMap(userId, remoteId string) (infos map[string]string, err error) IsLocalIdentity(userId, identityId string) bool IsRemoteIdentity(userId, identityId string) bool Close() } ) type IdentityStorageUpdater interface { UpdateUserIdentity(userIdentity *UserIdentity, fields map[string]interface{}) error } ================================================ FILE: src/backend/main/go.backends/KeysInterfaces.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package backends import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type KeysStorage interface { CreatePGPPubKey(pubkey *PublicKey) CaliopenError RetrieveContactPubKeys(userId, contactId string) (PublicKeys, CaliopenError) RetrievePubKey(userId, resourceId, keyId string) (*PublicKey, CaliopenError) DeletePubKey(pubkey *PublicKey) CaliopenError UpdatePubKey(newPubKey, oldPubKey *PublicKey, modifiedFields map[string]interface{}) CaliopenError } ================================================ FILE: src/backend/main/go.backends/LDAInterfaces.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backends import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "io" "time" ) //Local Delivery Agent storage interface type LDAStore interface { Close() RetrieveMessage(user_id, msg_id string) (msg *Message, err error) GetUsersForLocalMailRecipients([]string) ([][]UUID, error) // returns a list of tuples ([user_id, identity_id]) of **local** users found for given recipients list. No deduplicate. GetSettings(user_id string) (settings *Settings, err error) CreateMessage(msg *Message) error StoreRawMessage(msg RawMessage) (err error) GetRawMessage(raw_message_id string) (raw_message RawMessage, err error) SetDeliveredStatus(raw_msg_id string, delivered bool) error UpdateMessage(msg *Message, fields map[string]interface{}) error // 'fields' are the struct fields names that have been modified SeekMessageByExternalRef(userID, externalMessageID, identityID string) (UUID, error) LookupContactsByIdentifier(user_id, address, kind string) (contact_ids []string, err error) GetAttachment(uri string) (file io.Reader, err error) DeleteAttachment(uri string) error AttachmentExists(uri string) bool RetrieveUserIdentity(userId, identityId string, withCredentials bool) (*UserIdentity, error) UpdateUserIdentity(userIdentity *UserIdentity, fields map[string]interface{}) error RetrieveUser(user_id string) (user *User, err error) UpdateRemoteInfosMap(userId, remoteId string, infos map[string]string) error RetrieveRemoteInfosMap(userId, remoteId string) (infos map[string]string, err error) TimestampRemoteLastCheck(userId, remoteId string, time ...time.Time) error RetrieveProvider(name, instance string) (*Provider, CaliopenError) } type LDAIndex interface { Close() CreateMessage(user *UserInfo, msg *Message) error UpdateMessage(user *UserInfo, msg *Message, fields map[string]interface{}) error } ================================================ FILE: src/backend/main/go.backends/MessagesInterfaces.go ================================================ package backends import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type MessageStorage interface { CreateMessage(msg *Message) error RetrieveMessage(user_id, msg_id string) (msg *Message, err error) UpdateMessage(msg *Message, fields map[string]interface{}) error // 'fields' are the struct fields names that have been modified DeleteMessage(msg *Message) error SetMessageUnread(user_id, message_id string, status bool) error GetRawMessage(raw_message_id string) (raw_message RawMessage, err error) } type MessageIndex interface { SetMessageUnread(user *UserInfo, message_id string, status bool) error CreateMessage(user *UserInfo, msg *Message) error UpdateMessage(user *UserInfo, msg *Message, fields map[string]interface{}) error FilterMessages(search IndexSearch) (messages []*Message, totalFound int64, err error) GetMessagesRange(search IndexSearch) (messages []*Message, totalFound int64, err error) } ================================================ FILE: src/backend/main/go.backends/NotificationsInterfaces.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backends import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "time" ) type NotificationsStore interface { CreateMessage(msg *Message) error UserByUsername(username string) (user *User, err error) // to retrieve admin user RetrieveLocalsIdentities(user_id string) (identities []UserIdentity, err error) PutNotificationInQueue(*Notification) error NotificationsByTime(userId string, from, to time.Time) ([]Notification, error) NotificationsByID(userId, from, to string) ([]Notification, error) RetrieveNotification(userId, notificationId string) (Notification, error) DeleteNotifications(userId string, until time.Time) error DeleteNotification(userId, notificationId string) error } type NotificationsIndex interface { CreateMessage(user *UserInfo, msg *Message) error } ================================================ FILE: src/backend/main/go.backends/ProvidersInterfaces.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backends import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type ( ProviderStorage interface { CreateProvider(*Provider) CaliopenError RetrieveProvider(name, instance string) (*Provider, CaliopenError) UpdateProvider(*Provider, map[string]interface{}) CaliopenError DeleteProvider(*Provider) CaliopenError } ) ================================================ FILE: src/backend/main/go.backends/RESTInterfaces.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backends import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/gocql/gocql" ) type APIStorage interface { AttachmentStorage CredentialsStorage ContactStorage DevicesStorage DiscussionStorage IdentityStorage KeysStorage MessageStorage UrisStorage TagsStorage UserNameStorage UserStorage ProviderStorage GetSession() *gocql.Session } type APIIndex interface { MessageIndex ContactIndex DiscussionIndex RecipientsSuggest(user *UserInfo, query_string string) (suggests []RecipientSuggestion, err error) Search(search IndexSearch) (result *IndexResult, err error) } ================================================ FILE: src/backend/main/go.backends/TagsInterfaces.go ================================================ package backends import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type TagsStorage interface { RetrieveUserTags(user_id string) (tags []Tag, err error) CreateTag(tag *Tag) error RetrieveTag(user_id, tag_id string) (tag Tag, err error) UpdateTag(tag *Tag) error DeleteTag(user_id, tag_id string) error } ================================================ FILE: src/backend/main/go.backends/UrisInterface.go ================================================ package backends import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type UrisStorage interface { LookupHash(user_id UUID, uri string) ([]HashLookup, error) CreateHashLookup(lookup HashLookup) error } ================================================ FILE: src/backend/main/go.backends/UsersInterfaces.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backends import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type ( UserStorage interface { GetSettings(userId string) (settings *Settings, err error) RetrieveUser(userId string) (user *User, err error) UpdateUserPasswordHash(user *User) error UpdateUser(user *User, fields map[string]interface{}) error // 'fields' are the struct fields names that have been modified UserByRecoveryEmail(email string) (user *User, err error) DeleteUser(userId string) error GetShardForUser(userID string) string } UserNameStorage interface { UsernameIsAvailable(username string) (bool, error) UserByUsername(username string) (user *User, err error) } ) ================================================ FILE: src/backend/main/go.backends/backendstest/APIstore.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backendstest import ( "github.com/gocql/gocql" ) type APIStore struct { AttachmentStore CredentialStore ContactsBackend DevicesStore DiscussionsStore IdentitiesBackend KeysStore MessagesBackend ParticipantStore TagsStore UserNamesStore UsersBackend ProvidersStore } func (s *APIStore) GetSession() *gocql.Session { return nil } ================================================ FILE: src/backend/main/go.backends/backendstest/Attachments.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backendstest import ( "errors" "io" ) type AttachmentStore struct{} func (as AttachmentStore) StoreAttachment(attachment_id string, file io.Reader) (uri string, size int, err error) { return "", 0, errors.New("test interface not implemented") } func (as AttachmentStore) GetAttachment(uri string) (file io.Reader, err error) { return nil, errors.New("test interface not implemented") } func (as AttachmentStore) DeleteAttachment(uri string) error { return errors.New("test interface not implemented") } ================================================ FILE: src/backend/main/go.backends/backendstest/Cache.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backendstest import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "gopkg.in/redis.v5" "time" ) type MockRedis struct { Store map[string][]byte Ttl map[string]time.Duration } func (mr *MockRedis) GetAuthToken(token string) (value *Auth_cache, err error) { return nil, errors.New("test interface not implemented") } func (mr *MockRedis) LogoutUser(key string) error { return errors.New("test interface not implemented") } func (mr *MockRedis) GetResetPasswordToken(token string) (*TokenSession, error) { return nil, errors.New("test interface not implemented") } func (mr *MockRedis) GetResetPasswordSession(user_id string) (*TokenSession, error) { return nil, errors.New("test interface not implemented") } func (mr *MockRedis) SetResetPasswordSession(user_id, reset_token string) (*TokenSession, error) { return nil, errors.New("test interface not implemented") } func (mr *MockRedis) DeleteResetPasswordSession(user_id string) error { return errors.New("test interface not implemented") } func (mr *MockRedis) SetOauthSession(key string, session *OauthSession) error { return errors.New("test interface not implemented") } func (mr *MockRedis) GetOauthSession(key string) (*OauthSession, error) { return nil, errors.New("test interface not implemented") } func (mr *MockRedis) DeleteOauthSession(user_id string) error { return errors.New("test interface not implemented") } func (mr *MockRedis) GetDeviceValidationSession(userId, deviceId string) (*TokenSession, error) { return nil, errors.New("test interface not implemented") } func (mr *MockRedis) GetTokenValidationSession(userId, token string) (*TokenSession, error) { return nil, errors.New("test interface not implemented") } func (mr *MockRedis) SetDeviceValidationSession(userId, deviceId, token string) (*TokenSession, error) { return nil, errors.New("test interface not implemented") } func (mr *MockRedis) DeleteDeviceValidationSession(userId, deviceId string) error { return errors.New("test interface not implemented") } // Set mocks Set func from gopkg.in/redis.v5/internal // expiration is not handled func (mr *MockRedis) Set(key string, value []byte, expiration time.Duration) error { mr.Store[key] = value mr.Ttl[key] = expiration return nil } // Get mocks Get func from gopkg.in/redis.v5/internal func (mr *MockRedis) Get(key string) (value []byte, err error) { if v, ok := mr.Store[key]; ok { return v, err } else { return nil, redis.Nil } } // Del mocks Del func from gopkg.in/redis.v5/internal func (mr *MockRedis) Del(key string) error { delete(mr.Store, key) return nil } // GetTTL returns the Ttl that has been set along with a key when Set has been previously called // for testing purpose func (mr *MockRedis) GetTTL(key string) (Ttl time.Duration, err error) { if v, ok := mr.Ttl[key]; ok { return v, err } else { return 0, errors.New("not found") } } ================================================ FILE: src/backend/main/go.backends/backendstest/Contacts.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backendstest import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type ContactsBackend struct { contacts map[string]*Contact } func GetContactBackend() ContactsBackend { return ContactsBackend{ contacts: Contacts, } } func (cb ContactsBackend) CreateContact(contact *Contact) error { return errors.New("CreateContact test interface not implemented") } func (cb ContactsBackend) RetrieveContact(userID, contactID string) (contact *Contact, err error) { return nil, errors.New("RetrieveContact test interface not implemented") } func (cb ContactsBackend) RetrieveUserContactId(userID string) string { return "" } func (cb ContactsBackend) UpdateContact(contact, oldContact *Contact, fields map[string]interface{}) error { //return errors.New("UpdateContact test interface not implemented") return nil } func (cb ContactsBackend) DeleteContact(contact *Contact) error { return errors.New("DeleteContact test interface not implemented") } func (cb ContactsBackend) ContactExists(userId, contactId string) bool { return false } func (cb ContactsBackend) LookupContactsByIdentifier(user_id, address, kind string) ([]string, error) { if contact_id, ok := ContactLookup[kind+":"+address]; ok { return []string{contact_id}, nil } else { return nil, errors.New("not found") } } func (cb ContactsBackend) ContactsForParticipants(userID string, participants map[string]Participant) error { return errors.New("ContactForParticipants test interface not implemented") } // ContactIndex interface type ContactsIndex struct { } func (ci ContactsIndex) CreateContact(user *UserInfo, contact *Contact) error { return errors.New("CreateContact test interface not implemented") } func (ci ContactsIndex) UpdateContact(user *UserInfo, contact *Contact, fields map[string]interface{}) error { //return errors.New("UpdateContact test interface not implemented") return nil } func (ci ContactsIndex) FilterContacts(search IndexSearch) (contacts []*Contact, totalFound int64, err error) { return nil, 0, errors.New("FilterContact test interface not implemented") } func (ci ContactsIndex) DeleteContact(user *UserInfo, contact *Contact) error { return errors.New("DeleteContact test interface not implemented") } ================================================ FILE: src/backend/main/go.backends/backendstest/Credentials.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backendstest import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type CredentialStore struct{} func (cs CredentialStore) CreateCredentials(userIdentity *UserIdentity, cred Credentials) error { return errors.New("test interface not implemented") } func (cs CredentialStore) RetrieveCredentials(userId, remoteId string) (Credentials, error) { return nil, errors.New("test interface not implemented") } func (cs CredentialStore) UpdateCredentials(userId, remoteId string, cred Credentials) error { return errors.New("test interface not implemented") } func (cs CredentialStore) DeleteCredentials(userId, remoteId string) error { return errors.New("test interface not implemented") } ================================================ FILE: src/backend/main/go.backends/backendstest/Devices.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backendstest import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type DevicesStore struct { devices map[string]*Device } func (ds DevicesStore) CreateDevice(device *Device) error { return errors.New("test interface not implemented") } func (ds DevicesStore) RetrieveDevices(user_id string) (devices []Device, err error) { return nil, errors.New("test interface not implemented") } func (ds DevicesStore) RetrieveDevice(userId, deviceId string) (device *Device, err error) { if device, ok := Devices[userId+deviceId]; ok { return device, nil } return nil, errors.New("not found") } func (ds DevicesStore) UpdateDevice(device, oldDevice *Device, modifiedFields map[string]interface{}) error { Devices[device.UserId.String()+device.DeviceId.String()] = device return nil } func (ds DevicesStore) DeleteDevice(device *Device) error { return errors.New("test interface not implemented") } ================================================ FILE: src/backend/main/go.backends/backendstest/Discussions.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backendstest import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type DiscussionsStore struct{} func (ds *DiscussionsStore) GetUserLookupHashes(userId UUID, kind, key string) (hashes []ParticipantHash, err error) { return nil, errors.New("test interface not implemented") } func (ds *DiscussionsStore) UpsertDiscussionLookups(userId UUID, participants []Participant) error { return errors.New("test interface not implemented") } ================================================ FILE: src/backend/main/go.backends/backendstest/Identities.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. // Package backendstest provides utilities and interfaces for mocking backends interfaces package backendstest import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) // IdentitiesStorage and IdentityStorageUpdater implementation type IdentitiesBackend struct { localIdentities map[string]*UserIdentity remoteIdentities map[string]*UserIdentity } // GetIdentitiesBackend returns an IdentitiesBackend implementing all IdentitiesStorage interfaces // serving default testdata unless some data are provided in params arrays func GetIdentitiesBackend(locals, remotes []*UserIdentity) *IdentitiesBackend { i := IdentitiesBackend{} if len(locals) > 0 { for _, local := range locals { i.localIdentities[local.UserId.String()+local.Id.String()] = local } } else { i.localIdentities = LocalIdentities } if len(remotes) > 0 { for _, remote := range remotes { i.remoteIdentities[remote.UserId.String()+remote.Id.String()] = remote } } else { i.remoteIdentities = RemoteIdentities } return &i } func (ib IdentitiesBackend) RetrieveLocalsIdentities(user_id string) ([]UserIdentity, error) { return RetrieveLocalsIdentities(user_id) } func (ib IdentitiesBackend) CreateUserIdentity(userIdentity *UserIdentity) CaliopenError { return NewCaliopenErr(NotImplementedCaliopenErr, "test interface not implemented") } func (ib IdentitiesBackend) RetrieveUserIdentity(userId, identityId string, withCredentials bool) (*UserIdentity, error) { return RetrieveUserIdentity(userId, identityId, withCredentials) } func (ib IdentitiesBackend) LookupIdentityByIdentifier(string, ...string) ([][2]string, error) { return [][2]string{}, errors.New("test interface not implemented") } func (ib IdentitiesBackend) LookupIdentityByType(string, ...string) ([][2]string, error) { return [][2]string{}, errors.New("test interface not implemented") } func (ib IdentitiesBackend) UpdateUserIdentity(userIdentity *UserIdentity, fields map[string]interface{}) error { return errors.New("test interface not implemented") } func (ib IdentitiesBackend) DeleteUserIdentity(userIdentity *UserIdentity) error { return errors.New("test interface not implemented") } func (ib IdentitiesBackend) RetrieveRemoteIdentities(userId string, withCredentials bool) ([]*UserIdentity, error) { return nil, errors.New("test interface not implemented") } func (ib IdentitiesBackend) RetrieveAllRemotes(withCredentials bool) (<-chan *UserIdentity, error) { return RetrieveAllRemotes(withCredentials) } func (ib IdentitiesBackend) UpdateRemoteInfosMap(userId, remoteId string, infos map[string]string) error { return errors.New("test interface not implemented") } func (ib IdentitiesBackend) RetrieveRemoteInfosMap(userId, remoteId string) (infos map[string]string, err error) { return map[string]string{}, errors.New("test interface not implemented") } func (ib IdentitiesBackend) IsLocalIdentity(userId, identityId string) bool { return false } func (ib IdentitiesBackend) IsRemoteIdentity(userId, identityId string) bool { return false } func (ib IdentitiesBackend) Close() { } func LocalsCount() int { return len(LocalIdentities) } func RemotesCount() int { return len(RemoteIdentities) } func ActiveRemotesCount() int { var c int for _, remote := range RemoteIdentities { if remote.Status == "active" { c += 1 } } return c } func RetrieveLocalsIdentities(user_id string) ([]UserIdentity, error) { locals := []UserIdentity{} for _, local := range LocalIdentities { locals = append(locals, *local) } return locals, nil } func RetrieveUserIdentity(userId, identityId string, withCredentials bool) (*UserIdentity, error) { if userId == "" && identityId == "" { // return one remote identity by default for _, remote := range RemoteIdentities { if !withCredentials { remote.Credentials = nil } return remote, nil } } if remote, ok := RemoteIdentities[userId+identityId]; ok { if !withCredentials { remote.Credentials = nil } return remote, nil } if local, ok := LocalIdentities[userId+identityId]; ok { if !withCredentials { local.Credentials = nil } return local, nil } return nil, errors.New("not found") } func RetrieveAllRemotes(withCredentials bool) (<-chan *UserIdentity, error) { ch := make(chan *UserIdentity) go func() { for _, remote := range RemoteIdentities { ch <- remote } close(ch) }() return ch, nil } ================================================ FILE: src/backend/main/go.backends/backendstest/Keys.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backendstest import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type KeysStore struct{} func (ks KeysStore) CreatePGPPubKey(pubkey *PublicKey) CaliopenError { return NewCaliopenErr(NotImplementedCaliopenErr, "test interface not implemented") } func (ks KeysStore) RetrieveContactPubKeys(userId, contactId string) (PublicKeys, CaliopenError) { return nil, NewCaliopenErr(NotImplementedCaliopenErr, "test interface not implemented") } func (ks KeysStore) RetrievePubKey(userId, resourceId, keyId string) (*PublicKey, CaliopenError) { return nil, NewCaliopenErr(NotImplementedCaliopenErr, "test interface not implemented") } func (ks KeysStore) DeletePubKey(pubkey *PublicKey) CaliopenError { return NewCaliopenErr(NotImplementedCaliopenErr, "test interface not implemented") } func (ks KeysStore) UpdatePubKey(newPubKey, oldPubKey *PublicKey, modifiedFields map[string]interface{}) CaliopenError { return NewCaliopenErr(NotImplementedCaliopenErr, "test interface not implemented") } ================================================ FILE: src/backend/main/go.backends/backendstest/LDA.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backendstest import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "io" "time" ) type LDAStoreBackend struct { } type LDAIndexBackend struct { } // GetLDAStoreBackend returns an LDAStoreBackend implementing all LDAStore interfaces // serving default testdata unless some data are provided in params arrays func GetLDAStoreBackend() *LDAStoreBackend { s := LDAStoreBackend{} return &s } // GetLDAIndexBackend returns an LDAIndexBackend implementing all LDAIndex interfaces // serving default testdata unless some data are provided in params arrays func GetLDAIndexBackend() *LDAIndexBackend { i := LDAIndexBackend{} return &i } func (ldaStore *LDAStoreBackend) Close() { } func (ldaStore *LDAStoreBackend) RetrieveMessage(userId, msgId string) (msg *Message, err error) { mb := GetMessagesBackend() return mb.RetrieveMessage(userId, msgId) } func (ldaStore *LDAStoreBackend) GetUsersForLocalMailRecipients([]string) ([][]UUID, error) { return nil, errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) GetSettings(userId string) (settings *Settings, err error) { return nil, errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) CreateMessage(msg *Message) (err error) { return errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) StoreRawMessage(msg RawMessage) (err error) { return errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) GetRawMessage(rawMsgId string) (rawMsg RawMessage, err error) { return RawMessage{}, errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) SetDeliveredStatus(raw_msg_id string, delivered bool) error { return errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) UpdateMessage(msg *Message, fields map[string]interface{}) error { return errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) CreateThreadLookup(user_id, discussion_id UUID, external_msg_id string) error { return errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) SeekMessageByExternalRef(userID, externalMessageID, identityID string) (UUID, error) { return EmptyUUID, errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) LookupContactsByIdentifier(user_id, address, kind string) (contact_ids []string, err error) { return nil, errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) GetAttachment(uri string) (file io.Reader, err error) { return nil, errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) DeleteAttachment(uri string) error { return errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) AttachmentExists(uri string) bool { return false } func (ldaStore *LDAStoreBackend) RetrieveUserIdentity(userId, identityId string, withCredentials bool) (*UserIdentity, error) { ib := GetIdentitiesBackend([]*UserIdentity{}, []*UserIdentity{}) return ib.RetrieveUserIdentity(userId, identityId, withCredentials) } func (ldaStore *LDAStoreBackend) UpdateUserIdentity(userIdentity *UserIdentity, fields map[string]interface{}) error { return errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) RetrieveUser(user_id string) (user *User, err error) { return nil, errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) UpdateRemoteInfosMap(userId, remoteId string, infos map[string]string) error { return errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) RetrieveRemoteInfosMap(userId, remoteId string) (infos map[string]string, err error) { return nil, errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) TimestampRemoteLastCheck(userId, remoteId string, time ...time.Time) error { return errors.New("test interface not implemented") } func (ldaStore *LDAStoreBackend) RetrieveProvider(name, instance string) (*Provider, CaliopenError) { return nil, NewCaliopenErr(NotImplementedCaliopenErr, "test interface not implemented") } func (ldIndex *LDAIndexBackend) Close() {} func (ldIndex *LDAIndexBackend) CreateMessage(user *UserInfo, msg *Message) error { return errors.New("test interface not implemented") } func (ldIndex *LDAIndexBackend) UpdateMessage(user *UserInfo, msg *Message, fields map[string]interface{}) error { return errors.New("test interface not implemented") } ================================================ FILE: src/backend/main/go.backends/backendstest/Messages.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backendstest import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type MessagesBackend map[string]*Message func GetMessagesBackend() MessagesBackend { return MessagesBackend(Msgs) } func (mb MessagesBackend) RetrieveMessage(userId, msgId string) (msg *Message, err error) { if userId == "" && msgId == "" { // return one message by default for _, msg = range mb { return } } var ok bool if msg, ok = mb[userId+msgId]; ok { return } return nil, errors.New("not found") } func (mb MessagesBackend) CreateMessage(msg *Message) error { return errors.New("test interface not implemented") } func (mb MessagesBackend) UpdateMessage(msg *Message, fields map[string]interface{}) error { return errors.New("test interface not implemented") } func (mb MessagesBackend) DeleteMessage(msg *Message) error { return errors.New("test interface not implemented") } func (mb MessagesBackend) SetMessageUnread(user_id, message_id string, status bool) error { return errors.New("test interface not implemented") } func (mb MessagesBackend) GetRawMessage(raw_message_id string) (raw_message RawMessage, err error) { return RawMessage{}, errors.New("test interface not implemented") } ================================================ FILE: src/backend/main/go.backends/backendstest/Notifications.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backendstest import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "time" ) type NotificationsStore struct{} type NotificationsIndex struct{} func GetNotificationsBackends() (store NotificationsStore, index NotificationsIndex) { return NotificationsStore{}, NotificationsIndex{} } func (ns NotificationsStore) CreateMessage(msg *Message) error { return errors.New("test interface not implemented") } func (ns NotificationsStore) UserByUsername(username string) (user *User, err error) { return UserByUsername(username) } func (ns NotificationsStore) RetrieveLocalsIdentities(userId string) (identities []UserIdentity, err error) { return RetrieveLocalsIdentities(userId) } func (ns NotificationsStore) PutNotificationInQueue(*Notification) error { return errors.New("test interface not implemented") } func (ns NotificationsStore) DeleteNotifications(userId string, until time.Time) error { return errors.New("test interface not implemented") } func (ns NotificationsStore) NotificationsByTime(userId string, from, to time.Time) ([]Notification, error) { return []Notification{}, errors.New("test interface not implemented") } func (ns NotificationsStore) NotificationsByID(userId, from, to string) ([]Notification, error) { return []Notification{}, errors.New("test interface not implemented") } func (ns NotificationsStore) RetrieveNotification(userId, notificationId string) (Notification, error) { return Notification{}, errors.New("test interface not implemented") } func (ns NotificationsStore) DeleteNotification(userId, notificationId string) error { return errors.New("test interface not implemented") } func (ni NotificationsIndex) CreateMessage(user *UserInfo, msg *Message) error { return errors.New("test interface not implemented") } ================================================ FILE: src/backend/main/go.backends/backendstest/Providers.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backendstest import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type ProvidersStore struct{} func (ps *ProvidersStore) CreateProvider(*Provider) CaliopenError { return NewCaliopenErr(NotImplementedCaliopenErr, "test interface not implemented") } func (ps *ProvidersStore) RetrieveProvider(name, instance string) (*Provider, CaliopenError) { return nil, NewCaliopenErr(NotImplementedCaliopenErr, "test interface not implemented") } func (ps *ProvidersStore) UpdateProvider(*Provider, map[string]interface{}) CaliopenError { return NewCaliopenErr(NotImplementedCaliopenErr, "test interface not implemented") } func (ps *ProvidersStore) DeleteProvider(*Provider) CaliopenError { return NewCaliopenErr(NotImplementedCaliopenErr, "test interface not implemented") } ================================================ FILE: src/backend/main/go.backends/backendstest/Tags.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backendstest import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type TagsStore struct{} func (ts TagsStore) RetrieveUserTags(user_id string) (tags []Tag, err error) { return nil, errors.New("test interface not implemented") } func (ts TagsStore) CreateTag(tag *Tag) error { return errors.New("test interface not implemented") } func (ts TagsStore) RetrieveTag(user_id, tag_id string) (tag Tag, err error) { return Tag{}, errors.New("test interface not implemented") } func (ts TagsStore) UpdateTag(tag *Tag) error { return errors.New("test interface not implemented") } func (ts TagsStore) DeleteTag(user_id, tag_id string) error { return errors.New("test interface not implemented") } ================================================ FILE: src/backend/main/go.backends/backendstest/UrisInterface.go ================================================ package backendstest import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type ParticipantStore struct { } func (ps *ParticipantStore) LookupHash(user_id UUID, uri string) ([]HashLookup, error) { return nil, errors.New("test interface not implemented") } func (ps *ParticipantStore) CreateHashLookup(lookup HashLookup) error { return errors.New("test interface not implemented") } ================================================ FILE: src/backend/main/go.backends/backendstest/UserNames.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backendstest import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type UserNamesStore struct{} func (uns UserNamesStore) UsernameIsAvailable(username string) (bool, error) { return false, errors.New("test interface not implemented") } func (uns UserNamesStore) UserByUsername(username string) (user *User, err error) { return UserByUsername(username) } func UserByUsername(username string) (user *User, err error) { for _, user := range Users { if user.Name == username { return user, nil } } return nil, errors.New("not found") } ================================================ FILE: src/backend/main/go.backends/backendstest/Users.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package backendstest import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type UsersBackend struct { users map[string]*User } func (ub UsersBackend) GetSettings(userID string) (settings *Settings, err error) { return nil, errors.New("GetSettings test interface not implemented") } func (ub UsersBackend) RetrieveUser(userID string) (user *User, err error) { if user, ok := Users[userID]; ok { return user, nil } return nil, errors.New("not found") } func (ub UsersBackend) UpdateUserPasswordHash(user *User) error { return errors.New("UpdateUserPasswordHash test interface not implemented") } func (ub UsersBackend) UpdateUser(user *User, fields map[string]interface{}) error { return errors.New("UpdateUser test interface not implemented") } func (ub UsersBackend) UserByRecoveryEmail(email string) (user *User, err error) { return nil, errors.New("UserByRecoveryEmail test interface not implemented") } func (ub UsersBackend) DeleteUser(userID string) error { return errors.New("DeleteUser test interface not implemented") } func (ub UsersBackend) GetShardForUser(userID string) string { if user, ok := Users[userID]; ok { return user.ShardId } return "" } ================================================ FILE: src/backend/main/go.backends/backendstest/testdata.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. // Package backendstest provides utilities and interfaces for mocking backends interfaces package backendstest import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/satori/go.uuid" "sync" "time" ) const ( DevIdoireUserId = "ede04443-b60f-4869-9040-20bd6b1e33c1" EmmaTommeUserId = "7f8329c4-e220-45fc-89b2-d8535df69e83" JeanThubUserId = "b26908c5-c32a-4301-8f79-80abf0d8f8fe" ) var ( Users = map[string]*User{ EmmaTommeUserId: { ContactId: UUID(uuid.FromStringOrNil("63ab7904-c416-4f1a-9652-3de82e4fd1f1")), FamilyName: "Tomme", GivenName: "Emma", Name: "emma", RecoveryEmail: "emma@recovery-caliopen.local", ShardId: "4faae137-5938-42d3-bf1a-8e1a4e1868e1", UserId: UUID(uuid.FromStringOrNil(EmmaTommeUserId)), }, DevIdoireUserId: { ContactId: UUID(uuid.FromStringOrNil("5f0baee8-1278-43eb-9931-01b7383b419b")), FamilyName: "Idoire", GivenName: "Dev", Name: "dev", RecoveryEmail: "dev@recovery-caliopen.local", ShardId: "4faae137-5938-42d3-bf1a-8e1a4e1868e1", UserId: UUID(uuid.FromStringOrNil(DevIdoireUserId)), }, } LocalIdentities = map[string]*UserIdentity{ DevIdoireUserId + "3fc38dde-1f11-42c0-a489-361d13caebac": { DisplayName: "Dev Idoire", Id: UUID(uuid.FromStringOrNil("3fc38dde-1f11-42c0-a489-361d13caebac")), Identifier: "idoire@caliopen.local", Infos: map[string]string{ "lastseenuid": "", "lastsync": "", "inserver": "", "outserver": "", "uidvalidity": "", "pollinterval": "15", }, LastCheck: time.Now(), Protocol: "email", Status: "active", Type: "local", UserId: UUID(uuid.FromStringOrNil(DevIdoireUserId)), }, EmmaTommeUserId + "cd1e6f68-163b-4fe6-8107-f10d140d3c35": { DisplayName: "Emma Tomme", Id: UUID(uuid.FromStringOrNil("cd1e6f68-163b-4fe6-8107-f10d140d3c35")), Identifier: "emmatomme@caliopen.local", Infos: map[string]string{ "lastseenuid": "", "lastsync": "", "inserver": "", "outserver": "", "uidvalidity": "", "pollinterval": "15", }, LastCheck: time.Now(), Protocol: "email", Status: "active", Type: "local", UserId: UUID(uuid.FromStringOrNil(EmmaTommeUserId)), }, JeanThubUserId + "6817de1c-0cc5-4964-8c47-58699cf783f7": { DisplayName: "Jean Thube", Id: UUID(uuid.FromStringOrNil("6817de1c-0cc5-4964-8c47-58699cf783f7")), Identifier: "jeanthube@caliopen.local", Infos: map[string]string{ "lastseenuid": "", "lastsync": "", "inserver": "", "outserver": "", "uidvalidity": "", "pollinterval": "15", }, LastCheck: time.Now(), Protocol: "email", Status: "active", Type: "local", UserId: UUID(uuid.FromStringOrNil(JeanThubUserId)), }, } RemoteIdentities = map[string]*UserIdentity{ DevIdoireUserId + "7e356efb-d24c-493a-b558-e58c7ad20ac3": { DisplayName: "Dev Idoire email remote", Id: UUID(uuid.FromStringOrNil("7e356efb-d24c-493a-b558-e58c7ad20ac3")), Identifier: "idoire@remote.server", Infos: map[string]string{ "lastseenuid": "", "lastsync": "", "inserver": "", "outserver": "", "uidvalidity": "", "pollinterval": "", }, LastCheck: time.Now(), Protocol: "email", Status: "active", Type: "remote", UserId: UUID(uuid.FromStringOrNil(DevIdoireUserId)), }, EmmaTommeUserId + "7e4eb26d-1b70-4bb3-b556-6c54f046e88e": { Credentials: &Credentials{ "oauh2refreshtoken": "1/MiibIppEIP0LtCxLpbdseVGNYrIqp0JtMwppeRMgbM5", // fake token "oauth2accesstoken": "bu32.sLujBrJwWQAoXJ4QMqdAgEOjiBfXu104dgI3fRDBY0bd-KuKeI1f4orAtMoMeBFFf1aJD6F9SEYo2p0hFWSOieyo-ASEqrJ38T4booBAIuWdV2sSMFw8n2bjNasa", // fake token "tokenexpiry": "2019-02-01T16:42:29+01:00", "tokentype": "Bearer", "username": "emmatomme@gmail.com"}, DisplayName: "Emma Tomme gmail", Id: UUID(uuid.FromStringOrNil("7e4eb26d-1b70-4bb3-b556-6c54f046e88e")), Identifier: "emmatomme@gmail.com", Infos: map[string]string{ "lastseenuid": "", "lastsync": "", "inserver": "", "outserver": "", "uidvalidity": "", "pollinterval": "5", }, LastCheck: time.Now(), Protocol: "email", Status: "active", Type: "remote", UserId: UUID(uuid.FromStringOrNil(EmmaTommeUserId)), }, EmmaTommeUserId + "b91f0fa8-17a2-4729-8a5a-5ff58ee5c121": { Credentials: &Credentials{ "secret": "b65ebjh9ACNFlhYwCByl0PEEAyU3wtNqOapplEwWuUEBl", // fake secret "token": "8977654370-cooB2ALLcP4OaejKk7g4lstpommuapp3Kki3dIU", // fake token }, DisplayName: "Emma Tomme twitter remote", Id: UUID(uuid.FromStringOrNil("b91f0fa8-17a2-4729-8a5a-5ff58ee5c121")), Identifier: "emmatomme", Infos: map[string]string{ "lastseenuid": "", "lastsync": "", "inserver": "", "outserver": "", "uidvalidity": "", "pollinterval": "5", "twitterid": "000000", }, LastCheck: time.Now(), Protocol: "twitter", Status: "active", Type: "remote", UserId: UUID(uuid.FromStringOrNil(EmmaTommeUserId)), }, JeanThubUserId + "a87b1a18-23e7-6744-8a5a-3ee71ee5c001": { DisplayName: "Jean Thube inactive remote", Id: UUID(uuid.FromStringOrNil("a87b1a18-23e7-6744-8a5a-3ee71ee5c001")), Identifier: "jeanthube", Infos: map[string]string{ "lastseenuid": "", "lastsync": "", "inserver": "", "outserver": "", "uidvalidity": "", "pollinterval": "5", }, LastCheck: time.Now(), Protocol: "twitter", Status: "inactive", Type: "remote", UserId: UUID(uuid.FromStringOrNil(JeanThubUserId)), }, } Contacts = map[string]*Contact{ DevIdoireUserId + "5f0baee8-1278-43eb-9931-01b7383b419b": { ContactId: UUID(uuid.FromStringOrNil("5f0baee8-1278-43eb-9931-01b7383b419b")), Emails: []EmailContact{ { Address: "dev@recovery-caliopen.local", EmailId: UUID(uuid.FromStringOrNil("64f782ef-721a-487a-a68a-d0a37707d463")), IsPrimary: false, Label: "dev@recovery-caliopen.local", Type: "other", }, }, FamilyName: "Idoire", GivenName: "Dev", Title: "Dev Idoire", UserId: UUID(uuid.FromStringOrNil(DevIdoireUserId)), }, EmmaTommeUserId + "63ab7904-c416-4f1a-9652-3de82e4fd1f1": { ContactId: UUID(uuid.FromStringOrNil("63ab7904-c416-4f1a-9652-3de82e4fd1f1")), Emails: []EmailContact{ { Address: "emma@recovery-caliopen.local", EmailId: UUID(uuid.FromStringOrNil("444d71f6-324c-4733-88a2-77ca28ea6d2d")), IsPrimary: false, Label: "emma@recovery-caliopen.local", Type: "other", }, }, Identities: []SocialIdentity{ { Name: "emmatoclite", Type: "twitter", }, }, FamilyName: "Tomme", GivenName: "Emma", Title: "Emma Tomme", UserId: UUID(uuid.FromStringOrNil(EmmaTommeUserId)), }, } Msgs = map[string]*Message{ EmmaTommeUserId + "b26e5ba4-34cc-42bb-9b70-5279648134f8": { Body_plain: "email's body plain", Is_draft: false, Message_id: UUID(uuid.FromStringOrNil("b26e5ba4-34cc-42bb-9b70-5279648134f8")), Raw_msg_id: UUID(uuid.FromStringOrNil("70beae6e-d96e-456e-9d78-7c13f00f0edd")), Subject: "Sent email message with external identity", User_id: UUID(uuid.FromStringOrNil(EmmaTommeUserId)), UserIdentities: []UUID{UUID(uuid.FromStringOrNil("7e4eb26d-1b70-4bb3-b556-6c54f046e88e"))}, }, } Devices = map[string]*Device{ EmmaTommeUserId + "b8c11acd-a90d-467f-90f7-21b6b615149d": { Locker: new(sync.Mutex), DeviceId: UUID(uuid.FromStringOrNil("b8c11acd-a90d-467f-90f7-21b6b615149d")), Name: "fake device for tests", Status: DeviceUnverifiedStatus, Type: DeviceLaptopType, UserId: UUID(uuid.FromStringOrNil(EmmaTommeUserId)), }, } // used to return contact by uris (kind + ":" + address) ContactLookup = map[string]string{ "email:dev@recovery-caliopen.local": "5f0baee8-1278-43eb-9931-01b7383b419b", "email:emma@recovery-caliopen.local": "63ab7904-c416-4f1a-9652-3de82e4fd1f1", "twitter:emmatomme": "63ab7904-c416-4f1a-9652-3de82e4fd1f1", } ) ================================================ FILE: src/backend/main/go.backends/cache/authentication.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package cache import ( "encoding/json" "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "strings" ) // GetAuthToken retrieves auth values stored for the given key // values are casted into an Auth_cache struct // key is in the form of "tokens::user_id" func (c *Cache) GetAuthToken(key string) (value *Auth_cache, err error) { value = &Auth_cache{} cache_str, err := c.Backend.Get(key) if err != nil { log.WithError(err).Errorf("[GetAuthToken] failed to get cache key %s", key) return nil, err } err = json.Unmarshal(cache_str, value) if err != nil { log.WithError(err).Errorf("[GetAuthToken] failed to unmarshal cache %s for key %s", cache_str, key) return nil, err } return } // LogoutUser will delete the entry of the user corresponding to the key func (c *Cache) LogoutUser(key string) error { if !strings.HasPrefix(key, "tokens::") { return errors.New("Unvalid key") } err := c.Backend.Del(key) if err != nil { log.WithError(err).Errorf("[LogoutUser] failed to delete key %s", key) } return err } ================================================ FILE: src/backend/main/go.backends/cache/cache.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package cache import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" ) type Cache struct { // il faut passer ça partout et juste set le backend en un redis backend // comme ça je pourrai créé un cachebackend avec un autre underlying backend CacheConfig Backend backends.CacheBackend } ================================================ FILE: src/backend/main/go.backends/cache/devicevalidation.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package cache import ( "encoding/json" "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "gopkg.in/redis.v5" "time" ) const ( validationPrefix = "validationsession::" deviceValidationTTL = 24 // ttl in hours ) func (c *Cache) GetDeviceValidationSession(userId, deviceId string) (session *TokenSession, err error) { return c.getValidationSession(validationPrefix + userId + "::" + deviceId) } func (c *Cache) GetTokenValidationSession(userId, token string) (session *TokenSession, err error) { return c.getValidationSession(validationPrefix + userId + "::" + token) } func (c *Cache) getValidationSession(key string) (session *TokenSession, err error) { session_str, err := c.Backend.Get(key) if err != nil { log.WithError(err).Errorf("[getValidationSession] failed to get key %s", key) return nil, err } session = &TokenSession{} err = json.Unmarshal(session_str, session) if err != nil { log.WithError(err).Errorf("[getValidationSession] failed to unmarshal value %s for key %s", session_str, key) return nil, err } return } // SetDeviceValidationSession sets two keys in cache facility // - one to retrieve session by device id // - one to retrieve session by token func (c *Cache) SetDeviceValidationSession(userId, deviceId, token string) (session *TokenSession, err error) { ttl := deviceValidationTTL * time.Hour expiration := time.Now().Add(ttl) session = &TokenSession{ ExpiresAt: expiration, ExpiresIn: int(ttl / time.Second), Token: token, UserId: userId, ResourceId: deviceId, } session_str, err := json.Marshal(session) if err != nil { log.WithError(err).Errorf("[SetDeviceValidationSession] failed to marshal session %+v", *session) return nil, err } prefix := validationPrefix + userId + "::" deviceKey := prefix + deviceId tokenKey := prefix + token err = c.Backend.Set(deviceKey, session_str, ttl) if err != nil { log.WithError(err).Errorf("[SetDeviceValidationSession] failed to set session key in cache for user %s, deviceId %s", userId, deviceId) return nil, err } err = c.Backend.Set(tokenKey, session_str, ttl) if err != nil { log.WithError(err).Errorf("[SetDeviceValidationSession] failed to set session key in cache for user %s, token %s", userId, token) _ = c.Backend.Del(deviceKey) return nil, err } return session, nil } // DeleteDeviceValidationSession deletes the two keys associated with a device validation session func (c *Cache) DeleteDeviceValidationSession(userId, deviceId string) error { session, _ := c.GetDeviceValidationSession(userId, deviceId) if session == nil { log.Errorf("[DeleteDeviceValidationSession] failed to retrieve session for user %s, device %s", userId, deviceId) return errors.New("not found") } prefix := validationPrefix + userId + "::" deviceKey := prefix + deviceId tokenKey := prefix + session.Token err := c.Backend.Del(deviceKey) if err != nil && err != redis.Nil { log.WithError(err).Errorf("[DeleteDeviceValidationSession] failed to delete device validation session for user %s, device %s", userId, deviceId) } err = c.Backend.Del(tokenKey) if err != nil && err != redis.Nil { log.WithError(err).Errorf("[DeleteDeviceValidationSession] failed to delete device validation session for user %s, token %s", userId, session.Token) } return nil } ================================================ FILE: src/backend/main/go.backends/cache/devicevalidation_test.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package cache import ( "bytes" "testing" "time" ) func TestRedisBackend_SetDeviceValidationSession(t *testing.T) { mockCache, mockRedis, err := InitializeTestCache() if err != nil { t.Error(err) return } session, err := mockCache.SetDeviceValidationSession("user_id", "device_id", "token") if err != nil { t.Error(err) } // check that TokenSession is well-formed ttl := int((deviceValidationTTL * time.Hour) / time.Second) if session.ExpiresIn != ttl { t.Errorf("expected TokenSession.ExpiresIn = %d, got %d", ttl, session.ExpiresIn) } if session.Token != "token" { t.Errorf("expected TokenSession.Token = token, got %s", session.Token) } if session.UserId != "user_id" { t.Errorf("expected TokenSession.UserId = user_id, got %s", session.UserId) } if session.ResourceId != "device_id" { t.Errorf("expected TokenSession.ResourceId = device_id, got ResourceId = %s", session.ResourceId) } // check that two keys with correct TTL have been effectively put in cache deviceValue, err := mockCache.Backend.Get(validationPrefix + "user_id::device_id") if err != nil { t.Errorf("failed to retrieve deviceKey : %s", err) } tokenValue, err := mockCache.Backend.Get(validationPrefix + "user_id::token") if err != nil { t.Errorf("failed to retrieve tokenKey : %s", err) } if !bytes.Equal(deviceValue, tokenValue) { t.Errorf("expected deviceValue and tokenValue to be the same, got %s and %s", string(deviceValue), string(tokenValue)) } deviceTTL, err := mockRedis.GetTTL(validationPrefix + "user_id::device_id") if err != nil { t.Errorf("failed to retrieve TTL for deviceKey : %s", err) } if int(deviceTTL/time.Second) != ttl { t.Errorf("expected deviceTTl set to %d, got %d", ttl, deviceTTL) } tokenTTL, err := mockRedis.GetTTL(validationPrefix + "user_id::token") if err != nil { t.Errorf("failed to retrieve TTL for tokenKey : %s", err) } if int(tokenTTL/time.Second) != ttl { t.Errorf("expected tokenTTl set to %d, got %d", ttl, tokenTTL) } } func TestRedisBackend_GetDeviceValidationSession(t *testing.T) { mockCache, _, err := InitializeTestCache() if err != nil { t.Error(err) return } sessionSet, err := mockCache.SetDeviceValidationSession("user_id", "device_id", "token") if err != nil { t.Error(err) } sessionGot, err := mockCache.GetDeviceValidationSession("user_id", "device_id") if err != nil { t.Error(err) } if sessionSet.Token != sessionGot.Token { t.Errorf("expected to retrieve a TokenSession with same Token = token, got %s", sessionGot.Token) } } func TestRedisBackend_GetTokenValidationSession(t *testing.T) { mockCache, _, err := InitializeTestCache() if err != nil { t.Error(err) return } sessionSet, err := mockCache.SetDeviceValidationSession("user_id", "device_id", "token") if err != nil { t.Error(err) } sessionGot, err := mockCache.GetTokenValidationSession("user_id", "device_id") if err != nil { t.Error(err) } if sessionSet.Token != sessionGot.Token { t.Errorf("expected to retrieve a TokenSession with same Token = token, got %s", sessionGot.Token) } } func TestRedisBackend_DeleteDeviceValidationSession(t *testing.T) { mockCache, mockRedis, err := InitializeTestCache() if err != nil { t.Error(err) return } _, err = mockCache.SetDeviceValidationSession("user_id", "device_id", "token") if err != nil { t.Error(err) } err = mockCache.DeleteDeviceValidationSession("user_id", "device_id") if err != nil { t.Error(err) } // check that both keys have been deleted var value []byte value, err = mockRedis.Get(validationPrefix + "user_id::device_id") if err == nil || value != nil { t.Errorf("delete deviceKey failed : expected to have not found error and nil value, got err = %s and value = %s", err, string(value)) } value, err = mockRedis.Get(validationPrefix + "user_id::token") if err == nil || value != nil { t.Errorf("delete tokenKey failed : expected to have not found error and nil value, got err = %s and value = %s", err, string(value)) } } ================================================ FILE: src/backend/main/go.backends/cache/oauthsessions.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package cache import ( "encoding/json" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "time" ) const ( oauthSessionPrefix = "oauthsession::" oauthSessionTTL = 10 // ttl in minutes ) // GetOauthSession unmarshal json found at `key`, if any, into an OauthSession struct func (c *Cache) GetOauthSession(key string) (session *OauthSession, err error) { session_str, err := c.Backend.Get(oauthSessionPrefix + key) if err != nil || len(session_str) == 0 { log.WithError(err).Errorf("[GetOauthSession] failed to get session with key %s", key) return nil, err } session = &OauthSession{} err = json.Unmarshal(session_str, session) if err != nil { log.WithError(err).Errorf("[GetOauthSession] failed to unmarshal session %s for key %s", session_str, key) return nil, err } return } // SetOauthSession put `OauthSession` as a json string at `key` prefixed with oauthSessionPrefix func (c *Cache) SetOauthSession(key string, session *OauthSession) (err error) { ttl := oauthSessionTTL * time.Minute session_str, err := json.Marshal(session) if err != nil { log.WithError(err).Errorf("[SetOauthSession] failed to marshal session.for key %s", key) return err } err = c.Backend.Set(oauthSessionPrefix+key, session_str, ttl) if err != nil { log.WithError(err).Errorf("[SetOauthSession] failed to set session for key %s", key) return err } return nil } // DeleteOauthSession deletes value found at `key` prefixed with oauthSessionPrefix func (c *Cache) DeleteOauthSession(key string) error { err := c.Backend.Del(oauthSessionPrefix + key) if err != nil { log.WithError(err).Errorf("[DeleteOauthSession] failed to delete session for key %s", key) return err } return nil } ================================================ FILE: src/backend/main/go.backends/cache/passwordreset.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package cache import ( "encoding/json" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "time" ) const ( sessionPrefix = "resetsession::" resetTokenPrefix = "resettoken::" resetPasswordTTL = 8 // ttl in hours ) // GetResetPasswordSession returns reset password session values stored for the userId, if any // Returns a nil 'session' if key is not found func (c *Cache) GetResetPasswordSession(userId string) (session *TokenSession, err error) { key := sessionPrefix + userId session_str, err := c.Backend.Get(key) if err != nil { log.WithError(err).Errorf("[GetResetPasswordSession] failed to get key %s", key) return nil, err } session = &TokenSession{} err = json.Unmarshal(session_str, session) if err != nil { log.WithError(err).Errorf("[GetResetPasswordSession] failed to unmarshal value %s for key %s", session_str, key) return nil, err } return } // SetResetPasswordSession stores key,value for given userId and resetToken. // The key will be in the form of "resetsession::userId". // Func will also call setResetPasswordToken() to add a secondary key in the form "resettoken::resetToken" pointing to the same value // Func returns a pointer to the Pass_reset_session object that represents values stored in the cache. // userId and resetToken strings must be well-formatted, they will not be checked. func (c *Cache) SetResetPasswordSession(userId, resetToken string) (session *TokenSession, err error) { ttl := resetPasswordTTL * time.Hour expiration := time.Now().Add(ttl) session = &TokenSession{ ExpiresAt: expiration, ExpiresIn: int(ttl / time.Second), Token: resetToken, UserId: userId, } session_str, err := json.Marshal(session) if err != nil { log.WithError(err).Errorf("[SetResetPasswordSession] failed to marshal session %+v", *session) return nil, err } err = c.Backend.Set(sessionPrefix+userId, session_str, ttl) if err != nil { log.WithError(err).Errorf("[SetResetPasswordSession] failed to set session key in cache for user %s", userId) return nil, err } err = c.setResetPasswordToken(resetToken, session_str, ttl) if err != nil { log.WithError(err).Errorf("[SetResetPasswordSession] failed to setResetPasswordToken in cache for user %s", userId) return nil, err } return session, nil } // SetResetPasswordToken stores key,value for given resetToken. // It is called by SetResetPasswordSession to add a secondary key pointing to the same underlying value. // The key is in the form "resettoken::resetToken" func (c *Cache) setResetPasswordToken(token string, session []byte, ttl time.Duration) error { return c.Backend.Set(resetTokenPrefix+token, session, ttl) } // GetResetPasswordToken returns values found for the given resetToken key func (c *Cache) GetResetPasswordToken(token string) (session *TokenSession, err error) { key := resetTokenPrefix + token session_str, err := c.Backend.Get(key) if err != nil { log.WithError(err).Errorf("[GetResetPasswordToken] failed to get key for token %s", token) return nil, err } session = &TokenSession{} err = json.Unmarshal(session_str, session) if err != nil { log.WithError(err).Errorf("[GetResetPasswordToken] failed to unmarshal session %s for token %s", session_str, token) return nil, err } return } // DeleteResetPasswordSession will delete two keys in a row : // the resetsession key and the resettoken key func (c *Cache) DeleteResetPasswordSession(userId string) error { session, err := c.GetResetPasswordSession(userId) if err != nil { log.WithError(err).Errorf("[DeleteResetPasswordSession] failed to get session for user %s", userId) return err } key := sessionPrefix + userId err = c.Backend.Del(key) if err != nil { log.WithError(err).Errorf("[DeleteResetPasswordSession] failed to delete session for user %s", userId) return err } key = resetTokenPrefix + session.Token err = c.Backend.Del(key) if err != nil { log.WithError(err).Errorf("[DeleteResetPasswordSession] failed to delete session token for user %s", userId) return err } return nil } ================================================ FILE: src/backend/main/go.backends/cache/redis.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package cache import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/backendstest" log "github.com/Sirupsen/logrus" "gopkg.in/redis.v5" "time" ) type redisBackend struct { client *redis.Client } func InitializeRedisBackend(config CacheConfig) (c *Cache, err error) { c = new(Cache) c.CacheConfig = config redisClient := redis.NewClient(&redis.Options{ Addr: config.Host, Password: config.Password, DB: config.Db, }) _, err = redisClient.Ping().Result() if err != nil { log.WithError(err).Errorf("[RedisBackend] initialize failed") return nil, err } c.Backend = &redisBackend{ client: redisClient, } return } func (rb *redisBackend) Set(key string, value []byte, ttl time.Duration) error { return rb.client.Set(key, value, ttl).Err() } func (rb *redisBackend) Get(key string) (value []byte, err error) { return rb.client.Get(key).Bytes() } func (rb *redisBackend) Del(key string) error { return rb.client.Del(key).Err() } func InitializeTestCache() (c *Cache, mock *backendstest.MockRedis, err error) { c = new(Cache) mock = &backendstest.MockRedis{ Store: map[string][]byte{}, Ttl: map[string]time.Duration{}, } c.Backend = mock return } ================================================ FILE: src/backend/main/go.backends/index/elasticsearch/broad_search.go ================================================ package index import ( "context" "encoding/json" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "github.com/satori/go.uuid" "gopkg.in/olivere/elastic.v5" ) // Composes a full text ES query from IndexSearch object, // making use of "common terms query" (see https://www.elastic.co/guide/en/elasticsearch/reference/5.4/query-dsl-common-terms-query.html). // The func returns a compound response from ES to return 5 relevant docs filed by type if no doctype is provided, // otherwise, all docs found within type are returned. // See search API readme file into doc folder to see how the search func could be used by frontend. func (es *ElasticSearchBackend) Search(search IndexSearch) (result *IndexResult, err error) { const ( sub_agg_key = "top_score_hits" agg_key = "by_type" ) q := elastic.NewBoolQuery() // Strictly filter on user_id q = q.Filter(elastic.NewTermQuery("user_id", search.User_id)) cutoffFrequency := 0.05 //words that have a document frequency greater than xx% will be treated as common terms. for field, value := range search.Terms { q = q.Should(elastic.NewCommonTermsQuery(field, value).CutoffFrequency(cutoffFrequency)) // always add the common fields below to improve results q = q.Should(elastic.NewCommonTermsQuery("body_plain", value).CutoffFrequency(cutoffFrequency)) q = q.Should(elastic.NewCommonTermsQuery("body_plain.normalized", value).CutoffFrequency(cutoffFrequency)) q = q.Should(elastic.NewCommonTermsQuery("body_html", value).CutoffFrequency(cutoffFrequency)) q = q.Should(elastic.NewCommonTermsQuery("body_html.normalized", value).CutoffFrequency(cutoffFrequency)) q = q.Should(elastic.NewCommonTermsQuery("subject", value).CutoffFrequency(cutoffFrequency)).Boost(2) q = q.Should(elastic.NewCommonTermsQuery("subject.normalized", value).CutoffFrequency(cutoffFrequency)).Boost(2) q = q.Should(elastic.NewCommonTermsQuery("given_name", value).CutoffFrequency(cutoffFrequency)).Boost(3) q = q.Should(elastic.NewCommonTermsQuery("given_name.normalized", value).CutoffFrequency(cutoffFrequency)).Boost(3) q = q.Should(elastic.NewCommonTermsQuery("family_name", value).CutoffFrequency(cutoffFrequency)).Boost(3) q = q.Should(elastic.NewCommonTermsQuery("family_name.normalized", value).CutoffFrequency(cutoffFrequency)).Boost(3) q = q.Should(elastic.NewCommonTermsQuery("title.raw", value).CutoffFrequency(cutoffFrequency)).Boost(5) q = q.Should(elastic.NewCommonTermsQuery("participants.address.raw", value).CutoffFrequency(cutoffFrequency)).Boost(2) q = q.Should(elastic.NewCommonTermsQuery("participants.label", value).CutoffFrequency(cutoffFrequency)).Boost(2) q = q.Should(elastic.NewCommonTermsQuery("emails.address.raw", value).CutoffFrequency(cutoffFrequency)).Boost(2) } // make aggregation to file docs by type: // get only the 5 most relevant doc for each type if search.DocType is empty // otherwise get docs according to limit & offset params. var s *elastic.SearchService switch search.DocType { case "": // no doctype provided. Trigger search on all document types within index and build an aggregation /*highlight disabled*/ //h := elastic.NewHighlight().Fields(elastic.NewHighlighterField("*").RequireFieldMatch(false)) top_hits := elastic.NewTopHitsAggregation().Size(5).FetchSource(true) //.Highlight(h) by_type := elastic.NewTermsAggregation().Field("_type").SubAggregation(sub_agg_key, top_hits) //TODO/WIP /*iq := elastic.NewIndicesQuery(elastic.NewRangeQuery("importance_level").Gte(search.ILrange[0]).Lte(search.ILrange[1]), MessageIndexType) msg_hits := elastic.NewFilterAggregation().Filter(iq) */ s = es.Client.Search().Index(search.Shard_id).Query(q).FetchSource(false).Aggregation(agg_key, by_type) //.Highlight(h) case MessageIndexType: // The search focuses on message document type, no aggregation needed, but importance level apply /*highlight disabled*/ //h := elastic.NewHighlight().Fields(elastic.NewHighlighterField("*").RequireFieldMatch(false)) rq := elastic.NewRangeQuery("importance_level").Gte(search.ILrange[0]).Lte(search.ILrange[1]) s = es.Client.Search().Index(search.Shard_id).Query(q).FetchSource(true). /*.Highlight(h)*/ PostFilter(rq) case ContactIndexType: // The search focuses on contact document type, no aggregation needed and importance level not taken into account /*highlight disabled*/ //h := elastic.NewHighlight().Fields(elastic.NewHighlighterField("*").RequireFieldMatch(false)) s = es.Client.Search().Index(search.Shard_id).Query(q).FetchSource(true) //.Highlight(h) } //prepare search // add type, from & size params only if type is not empty if search.DocType != "" { s = s.Type(search.DocType) if search.Offset > 0 { s = s.From(search.Offset) } if search.Limit > 0 { s = s.Size(search.Limit) } } /** log the full json query to help development source, _ := q.Source() json_query, _ := json.Marshal(source) log.Infof("\nES query source: %s\n", json_query) /** end of log **/ // execute the search s.MinScore(0.1) response, err := s.Do(context.TODO()) if err != nil { return nil, err } // build IndexResult from ES response result = &IndexResult{ Total: response.TotalHits(), MessagesHits: MessageHits{0, []*IndexHit{}}, ContactsHits: ContactHits{0, []*IndexHit{}}, } if search.DocType != "" { // no aggregation, thus elastic returns a parsed json switch search.DocType { case MessageIndexType: result.MessagesHits.Total = response.TotalHits() for _, hit := range response.Hits.Hits { msg := new(Message) if err := json.Unmarshal(*hit.Source, msg); err != nil { log.Info(err) continue } msg_id, _ := uuid.FromString(hit.Id) msg.Message_id.UnmarshalBinary(msg_id.Bytes()) msg_hit := new(IndexHit) msg_hit.Id = msg.Message_id msg_hit.Score = *hit.Score msg_hit.Highlights = hit.Highlight msg_hit.Document = msg (*result).MessagesHits.Messages = append((*result).MessagesHits.Messages, msg_hit) } case ContactIndexType: result.ContactsHits.Total = response.TotalHits() for _, hit := range response.Hits.Hits { contact := new(Contact) if err := json.Unmarshal(*hit.Source, contact); err != nil { log.Info(err) continue } contact_id, _ := uuid.FromString(hit.Id) contact.ContactId.UnmarshalBinary(contact_id.Bytes()) contact_hit := new(IndexHit) contact_hit.Id = contact.ContactId contact_hit.Score = *hit.Score contact_hit.Highlights = hit.Highlight contact_hit.Document = contact (*result).ContactsHits.Contacts = append((*result).ContactsHits.Contacts, contact_hit) } } } else { // elastic returns buckets aggregation as a raw []byte, need to unmarshal by_types, _ := response.Aggregations[agg_key] // by_types is a *json.RawMessage var agg map[string]interface{} err = json.Unmarshal(*by_types, &agg) if err != nil { return nil, err } buckets := agg["buckets"].([]interface{}) for _, b := range buckets { bucket := b.(map[string]interface{}) total, _ := bucket["doc_count"].(float64) switch bucket["key"].(string) { case MessageIndexType: (*result).MessagesHits.Total = int64(total) case ContactIndexType: (*result).ContactsHits.Total = int64(total) } //go deeper within map to get documents and unmarshal them to our objects top_score_hits, _ := bucket[sub_agg_key].(map[string]interface{}) hits, _ := top_score_hits["hits"].(map[string]interface{}) hits_hits, _ := hits["hits"].([]interface{}) for _, hh := range hits_hits { hit := hh.(map[string]interface{}) h := new(IndexHit) id, _ := hit["_id"].(string) uid, _ := uuid.FromString(id) h.Id.UnmarshalBinary(uid.Bytes()) h.Score, _ = hit["_score"].(float64) highlights, _ := hit["highlight"].(map[string]interface{}) h.Highlights = map[string][]string{} for key, value := range highlights { for _, highlight := range value.([]interface{}) { h.Highlights[key] = append(h.Highlights[key], highlight.(string)) } } switch bucket["key"].(string) { case MessageIndexType: msg_map, _ := hit["_source"].(map[string]interface{}) msg := new(Message) if err := msg.UnmarshalMap(msg_map); err == nil { msg.User_id = search.User_id msg.Message_id = h.Id h.Document = msg (*result).MessagesHits.Messages = append(result.MessagesHits.Messages, h) } case ContactIndexType: contact_map, _ := hit["_source"].(map[string]interface{}) contact := new(Contact) if err := contact.UnmarshalMap(contact_map); err == nil { contact.UserId = search.User_id contact.ContactId = h.Id h.Document = contact (*result).ContactsHits.Contacts = append(result.ContactsHits.Contacts, h) } } } } } return } ================================================ FILE: src/backend/main/go.backends/index/elasticsearch/contacts.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package index import ( "context" "encoding/json" "errors" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "github.com/satori/go.uuid" "gopkg.in/oleiade/reflections.v1" "strings" ) func (es *ElasticSearchBackend) CreateContact(user *UserInfo, contact *Contact) error { es_contact, err := contact.MarshalES() if err != nil { log.WithError(err).Warnf("[ElasticSearchBackend] failed to parse contact to json : %s", string(es_contact)) return err } resp, err := es.Client.Index().Index(user.Shard_id).Type(ContactIndexType).Id(contact.ContactId.String()). BodyString(string(es_contact)). Refresh("wait_for"). Do(context.TODO()) if err != nil { log.WithError(err).Warnf("[ElasticSearchBackend] CreateContact failed for user %s and contact %s", contact.UserId.String(), contact.ContactId.String()) return err } log.Infof("New contact indexed with id %s", resp.Id) return nil } func (es *ElasticSearchBackend) UpdateContact(user *UserInfo, contact *Contact, fields map[string]interface{}) error { //get json field name for each field to modify jsonFields := map[string]interface{}{} for field, value := range fields { jsonField, err := reflections.GetFieldTag(contact, field, "json") if err != nil { return fmt.Errorf("[ElasticSearchBackend] UpdateContact failed to find a json field for object field %s", field) } split := strings.Split(jsonField, ",") jsonFields[split[0]] = value } update, err := es.Client.Update().Index(user.Shard_id).Type(ContactIndexType).Id(contact.ContactId.String()). Doc(jsonFields). Refresh("wait_for"). Do(context.TODO()) if err != nil { log.WithError(err).Warn("[ElasticSearchBackend] updateContact operation failed") return err } log.Infof("New version of indexed contact %s is now %d", update.Id, update.Version) return nil } func (es *ElasticSearchBackend) DeleteContact(user *UserInfo, contact *Contact) error { _, err := es.Client.Delete().Index(user.Shard_id).Type(ContactIndexType).Id(contact.ContactId.String()).Refresh("wait_for").Do(context.TODO()) if err != nil { return err } return nil } func (es *ElasticSearchBackend) SetContactUnread(user_id, Contact_id string, status bool) (err error) { return errors.New("[ElasticSearchBackend] not implemented") } func (es *ElasticSearchBackend) FilterContacts(filter IndexSearch) (contacts []*Contact, totalFound int64, err error) { search := es.Client.Search().Index(filter.Shard_id).Type(ContactIndexType) search = filter.FilterQuery(search, false).Sort("title.raw", true) if filter.Offset > 0 { search = search.From(filter.Offset) } if filter.Limit > 0 { search = search.Size(filter.Limit) } result, err := search.Do(context.TODO()) if err != nil { return nil, 0, err } for _, hit := range result.Hits.Hits { contact := new(Contact).NewEmpty().(*Contact) if err := json.Unmarshal(*hit.Source, contact); err != nil { log.Info(err) continue } contact_id, _ := uuid.FromString(hit.Id) contact.ContactId.UnmarshalBinary(contact_id.Bytes()) contact.UserId = filter.User_id contacts = append(contacts, contact) } totalFound = result.TotalHits() return } ================================================ FILE: src/backend/main/go.backends/index/elasticsearch/discussions.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package index import ( "context" "encoding/json" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/helpers" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/messages" uuid "github.com/satori/go.uuid" "gopkg.in/olivere/elastic.v5" ) // GetDiscussionList returns all the discussion_id found in index for an user // aggregated by discussion_id, with metadata. func (es *ElasticSearchBackend) GetDiscussionsList(filter IndexSearch, withIL bool) (discussions []Discussion, err error) { discussions = []Discussion{} search := es.Client.Search().Index(filter.Shard_id).Type(MessageIndexType) search = filter.FilterQuery(search, withIL) msgSource := elastic.NewFetchSourceContext(true) msgSource.Include("date_sort", "subject", "participants", "body_plain", "body_html", "importance_level") search.Aggregation("by_uris", elastic.NewTermsAggregation().Field("discussion_id").Size(10e3). // need to fetch all buckets to have an accurate buckets count SubAggregation("sub_bucket", elastic.NewTopHitsAggregation().Sort("date_sort", false).Size(1).FetchSourceContext(msgSource)). SubAggregation("unread_count", elastic.NewFilterAggregation().Filter(elastic.NewTermQuery("is_unread", true))). SubAggregation("importance_level", elastic.NewTermsAggregation().Field("importance_level").Size(100).OrderByTermDesc())) // TODO : attachment count aggr var result *elastic.SearchResult result, err = search.Do(context.TODO()) if err != nil { return nil, err } agg := result.Aggregations["by_uris"] var byUris Aggregation json.Unmarshal([]byte(*agg), &byUris) if len(byUris.Buckets) == 0 { return } /* byUris' Bucket.SubBucket : map[hits: map[hits: [map[ _id:xxxx _index:xxxx _score: _source:map[date_sort:xx participants:[xxxx] subject:xxx] */ // build discussions' array from raw bytes result for _, hit := range byUris.Buckets { msg := &Message{} var subBucket map[string]interface{} json.Unmarshal(hit.SubBucket, &subBucket) var subHit = subBucket["hits"].(map[string]interface{})["hits"].([]interface{})[0].(map[string]interface{}) msg.UnmarshalMap(subHit["_source"].(map[string]interface{})) // flatten importance levels distribution to an array of float64 for later computation var importanceAgg Aggregation json.Unmarshal(hit.ImportanceLevel, &importanceAgg) messagesIL := []float64{} for _, bucket := range importanceAgg.Buckets { for i := 0; i < bucket.DocCount; i++ { messagesIL = append(messagesIL, float64(bucket.Key.(float64))) } } discussions = append(discussions, Discussion{ DateUpdate: msg.Date_sort, DiscussionId: hit.Key.(string), Excerpt: messages.ExcerptMessage(*msg, 200, true, true), ImportanceLevel: int32(helpers.ComputeDiscussionIL(messagesIL)), LastMessageDate: msg.Date_sort, LastMessageId: UUID(uuid.FromStringOrNil(subHit["_id"].(string))), LastMessageSubject: msg.Subject, Participants: msg.Participants, Subject: msg.Subject, TotalCount: int32(hit.DocCount), UnreadCount: int32(hit.UnreadCount.DocCount), UserId: filter.User_id, }) } return discussions, err } ================================================ FILE: src/backend/main/go.backends/index/elasticsearch/elasticsearch.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package index import ( "encoding/json" log "github.com/Sirupsen/logrus" elastic "gopkg.in/olivere/elastic.v5" ) type ( ElasticSearchBackend struct { ElasticSearchConfig Client *elastic.Client } ElasticSearchConfig struct { Urls []string `mapstructure:"elastic_urls"` } Bucket struct { DocCount int `json:"doc_count"` Key interface{} `json:"key"` SubBucket json.RawMessage `json:"sub_bucket"` UnreadCount DocCounter `json:"unread_count"` ImportanceLevel json.RawMessage `json:"importance_level"` } Aggregation struct { Buckets []Bucket `json:"buckets"` DocCountErrorUpperBound int `json:"doc_count_error_upper_bound"` SumOtherDocCount int `json:"sum_other_doc_count"` } DocCounter struct { DocCount int `json:"doc_count"` } ) func InitializeElasticSearchIndex(config ElasticSearchConfig) (es *ElasticSearchBackend, err error) { es = new(ElasticSearchBackend) err = es.initialize(config) return } func (es *ElasticSearchBackend) initialize(config ElasticSearchConfig) (err error) { // Create elastic client es.Client, err = elastic.NewClient( elastic.SetURL(config.Urls...), //elastic.SetTraceLog(log.StandardLogger()), // comment out to stop tracing requests & responses ) if err != nil { log.WithError(err).Warn("package index : failed to create ES client") return } return } func (es *ElasticSearchBackend) Close() { es.Client.Stop() } ================================================ FILE: src/backend/main/go.backends/index/elasticsearch/messages.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package index import ( "context" "encoding/json" "errors" "fmt" objects "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "github.com/satori/go.uuid" "gopkg.in/oleiade/reflections.v1" "gopkg.in/olivere/elastic.v5" "sort" "strings" ) func (es *ElasticSearchBackend) CreateMessage(user *objects.UserInfo, msg *objects.Message) error { es_msg, err := msg.MarshalES() if err != nil { return err } resp, err := es.Client.Index().Index(user.Shard_id).Type(objects.MessageIndexType).Id(msg.Message_id.String()). BodyString(string(es_msg)). Refresh("wait_for"). Do(context.TODO()) if err != nil { log.WithError(err).Warn("backend Index: IndexMessage operation failed") return err } log.Infof("New msg indexed with id %s", resp.Id) return nil } func (es *ElasticSearchBackend) UpdateMessage(user *objects.UserInfo, msg *objects.Message, fields map[string]interface{}) error { //get json field name for each field to modify jsonFields := map[string]interface{}{} for field, value := range fields { jsonField, err := reflections.GetFieldTag(msg, field, "json") if err != nil { return fmt.Errorf("[ElasticSearchBackend] UpdateMessage failed to find a json field for object field %s", field) } split := strings.Split(jsonField, ",") jsonFields[split[0]] = value } update, err := es.Client.Update().Index(user.Shard_id).Type(objects.MessageIndexType).Id(msg.Message_id.String()). Doc(jsonFields). Refresh("wait_for"). Do(context.TODO()) if err != nil { log.WithError(err).Warn("backend Index: updateMessage operation failed") return err } log.Infof("New version of indexed msg %s is now %d", update.Id, update.Version) return nil } func (es *ElasticSearchBackend) SetMessageUnread(user *objects.UserInfo, message_id string, status bool) (err error) { payload := struct { Is_unread bool `json:"is_unread"` }{status} update := es.Client.Update().Index(user.Shard_id).Type(objects.MessageIndexType).Id(message_id) _, err = update.Doc(payload).Refresh("true").Do(context.TODO()) return } func (es *ElasticSearchBackend) FilterMessages(filter objects.IndexSearch) (messages []*objects.Message, totalFound int64, err error) { search := es.Client.Search().Index(filter.Shard_id).Type(objects.MessageIndexType) search = filter.FilterQuery(search, true).Sort("date_sort", false) if filter.Offset > 0 { search = search.From(filter.Offset) } if filter.Limit > 0 { search = search.Size(filter.Limit) } return executeMessagesQuery(search) } // GetMessagesRange build a `search_after` query to retrieve messages before and/or after a specific message within a discussion func (es *ElasticSearchBackend) GetMessagesRange(filter objects.IndexSearch) (messages []*objects.Message, totalFound int64, err error) { messages = []*objects.Message{} var msg *objects.Message // remove range[] and msg_id from terms msgId := filter.Terms["msg_id"][0] delete(filter.Terms, "msg_id") var wantBefore bool var wantAfter bool for _, param := range filter.Terms["range[]"] { if param == "before" { wantBefore = true } else if param == "after" { wantAfter = true } } delete(filter.Terms, "range[]") // retrieve message with msg_id because search_after will not return it // XXX chamal: need Userinfo filtering esMsg, esErr := es.Client.Get().Index(filter.Shard_id).Type(objects.MessageIndexType).Id(msgId).Do(context.TODO()) if esErr != nil { return nil, 0, esErr } if !esMsg.Found { return nil, 0, errors.New("not found") } msg = new(objects.Message).NewEmpty().(*objects.Message) if err := json.Unmarshal(*esMsg.Source, msg); err != nil { return nil, 0, err } if e := msg.Message_id.UnmarshalBinary(uuid.FromStringOrNil(msgId).Bytes()); e != nil { log.WithError(e).Warnf("failed to unmarshal %s", msgId) } messages = append(messages, msg) // prepare search // make search_after query for `after` param if wantAfter { searchAfter := es.Client.Search().Index(filter.Shard_id).Type(objects.MessageIndexType) if filter.Offset > 0 { searchAfter = searchAfter.From(filter.Offset) } if filter.Limit > 0 { searchAfter = searchAfter.Size(filter.Limit) } searchAfter = filter.FilterQuery(searchAfter, true).Sort("date_sort", false).Sort("_uid", false) searchAfter = searchAfter.SearchAfter(msg.Date_sort.UnixNano()/10e5, objects.MessageIndexType+"#"+msgId) after, afterTotal, afterErr := executeMessagesQuery(searchAfter) if afterErr != nil { return nil, 0, afterErr } messages = append(messages, after...) totalFound = afterTotal } // make search_after query for `before` param if wantBefore { searchBefore := es.Client.Search().Index(filter.Shard_id).Type(objects.MessageIndexType) if filter.Offset > 0 { searchBefore = searchBefore.From(filter.Offset) } if filter.Limit > 0 { searchBefore = searchBefore.Size(filter.Limit) } searchBefore = filter.FilterQuery(searchBefore, true).Sort("date_sort", true).Sort("_uid", true) searchBefore = searchBefore.SearchAfter(msg.Date_sort.UnixNano()/10e5, objects.MessageIndexType+"#"+msgId) before, beforeTotal, beforeErr := executeMessagesQuery(searchBefore) if beforeErr != nil { return nil, 0, beforeErr } messages = append(messages, before...) totalFound = beforeTotal } sort.Sort(objects.ByDateSortAsc(messages)) return messages, totalFound, nil } func executeMessagesQuery(search *elastic.SearchService) (messages []*objects.Message, totalFound int64, err error) { result, err := search.Do(context.TODO()) if err != nil { return nil, 0, err } for _, hit := range result.Hits.Hits { msg := new(objects.Message).NewEmpty().(*objects.Message) if err := json.Unmarshal(*hit.Source, msg); err != nil { log.Info(err) continue } msg_id, _ := uuid.FromString(hit.Id) msg.Message_id.UnmarshalBinary(msg_id.Bytes()) messages = append(messages, msg) } totalFound = result.TotalHits() return } ================================================ FILE: src/backend/main/go.backends/index/elasticsearch/user_recipients_lookup.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package index import ( "context" "encoding/json" "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "gopkg.in/olivere/elastic.v5" ) type ( returnedContact struct { Fname string `json:"family_name"` Gname string `json:"given_name"` Title string `json:"title"` } returnedParticipant struct { Address string `json:"address"` Contact_ids []string `json:"contact_ids"` Label string `json:"label"` Protocol string `json:"protocol"` Type string `json:"type"` } returnedMessage struct { Date string `json:"date"` Type string `json:"type"` } ) // RecipientsSuggest builds ES queries and responses for finding relevant recipients when an user compose a message func (es *ElasticSearchBackend) RecipientsSuggest(user *UserInfo, query_string string) (suggests []RecipientSuggestion, err error) { suggests = []RecipientSuggestion{} q_string := query_string // build nested queries for participants lookup queries := []elastic.Query{ elastic.NewPrefixQuery("participants.label", q_string), elastic.NewPrefixQuery("participants.address.raw", q_string), elastic.NewTermQuery("participants.address.parts", q_string), } participants_fields_q := elastic.NewBoolQuery().Should(queries...) participants_q := elastic.NewNestedQuery("participants", participants_fields_q) participants_q.InnerHit(elastic.NewInnerHit().Size(1)) // build queries for contact lookup queries = []elastic.Query{ elastic.NewPrefixQuery("given_name", q_string).Boost(3), elastic.NewPrefixQuery("given_name.normalized", q_string).Boost(3), elastic.NewPrefixQuery("family_name", q_string).Boost(3), elastic.NewPrefixQuery("family_name.normalized", q_string).Boost(3), } contact_name_q := elastic.NewBoolQuery().Should(queries...) queries = []elastic.Query{ elastic.NewPrefixQuery("emails.label", q_string).Boost(2), elastic.NewPrefixQuery("emails.address.raw", q_string).Boost(2), elastic.NewTermQuery("emails.address.parts", q_string).Boost(2), } nested_emails_q := elastic.NewBoolQuery().Should(queries...) emails_q := elastic.NewNestedQuery("emails", nested_emails_q) emails_q.InnerHit(elastic.NewInnerHit()) queries = []elastic.Query{ elastic.NewPrefixQuery("ims.address", q_string).Boost(2), elastic.NewPrefixQuery("ims.label", q_string).Boost(2), } nested_ims_q := elastic.NewBoolQuery().Should(queries...) ims_q := elastic.NewNestedQuery("ims", nested_ims_q) ims_q.InnerHit(elastic.NewInnerHit()) queries = []elastic.Query{ elastic.NewPrefixQuery("identities.name", q_string).Boost(2), elastic.NewPrefixQuery("identities.infos", q_string).Boost(2), // TODO: check if this query could find string within infos map } nested_socials_q := elastic.NewBoolQuery().Should(queries...) socials_q := elastic.NewNestedQuery("identities", nested_socials_q) socials_q.InnerHit(elastic.NewInnerHit()) // doc source pruning fsc := elastic.NewFetchSourceContext(true) fsc.Include("title") // run the query main_query := elastic.NewBoolQuery().Filter(elastic.NewTermQuery("user_id", user.User_id)). Should(participants_q, contact_name_q, emails_q, ims_q, socials_q) search := es.Client.Search(). Index(user.Shard_id). FetchSourceContext(fsc). Size(30). MinScore(0.1) /** log the full json query to help development source, _ := main_query.Source() json_query, _ := json.Marshal(source) log.Infof("\nES query source: %s\n", json_query) agg_source, _ := max_date_agg.Source() json_agg, _ := json.Marshal(agg_source) log.Infof("\nES aggregation source: %s\n", json_agg) /** end of log **/ result, err := search.Query(main_query).Do(context.TODO()) if err != nil { log.WithError(err).Warn("[Elasticsearch] failed to suggest participant.") return } participants_suggests := make(map[string]RecipientSuggestion) for _, hit := range result.Hits.Hits { switch hit.Type { case MessageIndexType: suggest, e := extractParticipantInfos(hit) if e != nil { log.WithError(e).Warnf("[Elasticsearch] failed to extract message participants") continue } //deduplicate if _, ok := participants_suggests[suggest.Address]; !ok { participants_suggests[suggest.Address] = suggest suggests = append(suggests, suggest) } case ContactIndexType: suggest, e := extractContactInfos(hit) if e != nil { log.WithError(e).Warnf("[Elasticsearch] failed to extract contact info") continue } suggests = append(suggests, suggest) default: suggest := RecipientSuggestion{ Source: "<" + hit.Type + ">", } suggests = append(suggests, suggest) } } return } func extractContactInfos(contact_hit *elastic.SearchHit) (suggest RecipientSuggestion, err error) { var contact returnedContact if e := json.Unmarshal(*contact_hit.Source, &contact); e != nil { err = errors.New("[ES RecipientSuggest] failed unmarshaling hit's source : " + e.Error()) return } suggest.Source = "contact" suggest.Label = contact.Title suggest.Contact_Id = contact_hit.Id return } func extractParticipantInfos(message_hit *elastic.SearchHit) (suggest RecipientSuggestion, err error) { if participants, ok := message_hit.InnerHits["participants"]; ok && len(participants.Hits.Hits) > 0 { inner_hit := message_hit.InnerHits["participants"].Hits.Hits[0] var participant returnedParticipant if e := json.Unmarshal(*inner_hit.Source, &participant); e != nil { err = errors.New("[ES RecipientSuggest] failed unmarshaling hit's source : " + e.Error()) return } suggest.Source = "participant" suggest.Label = participant.Label suggest.Address = participant.Address suggest.Protocol = participant.Protocol return } return } ================================================ FILE: src/backend/main/go.backends/store/cassandra/attachments.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package store import "io" func (cb *CassandraBackend) StoreAttachment(attachment_id string, file io.Reader) (uri string, size int, err error) { uri, s, err := cb.ObjectsStore.PutAttachment(attachment_id, file) return uri, int(s), err } func (cb *CassandraBackend) DeleteAttachment(uri string) error { return cb.ObjectsStore.RemoveObject(uri) } func (cb *CassandraBackend) GetAttachment(uri string) (file io.Reader, err error) { return cb.ObjectsStore.GetObject(uri) } func (cb *CassandraBackend) AttachmentExists(uri string) bool { info, err := cb.ObjectsStore.StatObject(uri) if err == nil && info.Err == nil { return true } return false } ================================================ FILE: src/backend/main/go.backends/store/cassandra/cassandra.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package store import ( "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/object_store" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/vault" log "github.com/Sirupsen/logrus" "github.com/gocassa/gocassa" "github.com/gocql/gocql" "time" ) type ( CassandraBackend struct { CassandraConfig Session *gocql.Session IKeyspace gocassa.KeySpace //gocassa keyspace interface ObjectsStore object_store.ObjectsStore Timeout time.Duration Vault vault.HVault } CassandraConfig struct { Hosts []string `mapstructure:"hosts"` Keyspace string `mapstructure:"keyspace"` Consistency gocql.Consistency `mapstructure:"consistency_level"` SizeLimit uint64 `mapstructure:"raw_size_limit"` // max size to store (in bytes) WithObjStore bool // whether to use an objects store service for objects above SizeLimit object_store.OSSConfig UseVault bool `mapstructure:"use_vault"` vault.HVaultConfig } HasTable interface { // GetTableInfos returns the table name and maps with couple [PropertyName]CassandryKeys GetTableInfos() (table string, partitionKeys map[string]string, collectionKeys map[string]string) UnmarshalCQLMap(input map[string]interface{}) } ) const DefaultTimeout = time.Second * 2 func InitializeCassandraBackend(config CassandraConfig) (cb *CassandraBackend, err error) { cb = new(CassandraBackend) err = cb.initialize(config) if err != nil { return nil, err } // objects store if config.WithObjStore { cb.ObjectsStore, err = object_store.InitializeObjectsStore(config.OSSConfig) if err != nil { log.Warn("[InitializeCassandraBackend] object store initialization failed") return nil, err } } // credentials store cb.UseVault = config.UseVault if cb.UseVault { cb.Vault, err = vault.InitializeVaultBackend(config.HVaultConfig) if err != nil { log.Warn("[InitializeCassandraBackend] vault initialization failed") return nil, err } } return } func (cb *CassandraBackend) initialize(config CassandraConfig) (err error) { cb.CassandraConfig = config cb.Timeout = DefaultTimeout // connect to the cluster cluster := gocql.NewCluster(cb.CassandraConfig.Hosts...) cluster.Keyspace = cb.Keyspace cluster.Consistency = cb.Consistency //try to get a Session const maxAttempts = 10 for i := 0; i < maxAttempts; i++ { cb.Session, err = cluster.CreateSession() if err != nil { log.WithError(err).Warn("package store : unable to create a session to cassandra. Retrying in 3 sec…") time.Sleep(3 * time.Second) } else { break } } if err != nil { return } connection := gocassa.NewConnection(gocassa.GoCQLSessionToQueryExecutor(cb.Session)) cb.IKeyspace = connection.KeySpace(cb.Keyspace) return } func (cb *CassandraBackend) Close() { cb.Session.Close() } func (cb *CassandraBackend) GetSession() *gocql.Session { return cb.Session } // SessionQuery is a wrapper to cb.Session.Query(…………) that re-init a Session if it has been closed func (cb *CassandraBackend) SessionQuery(stmt string, values ...interface{}) *gocql.Query { if cb.Session.Closed() { err := (*cb).initialize(cb.CassandraConfig) if err != nil { log.WithError(err).Warn("[CassandraBackend] SessionQuery failed to re-init a gocql Session") } } return cb.Session.Query(stmt, values...) } ================================================ FILE: src/backend/main/go.backends/store/cassandra/contacts.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package store import ( "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" log "github.com/Sirupsen/logrus" "github.com/gocassa/gocassa" "github.com/gocql/gocql" "gopkg.in/oleiade/reflections.v1" ) // CreateContact saves Contact to Cassandra // AND fills/updates joined and lookup tables func (cb *CassandraBackend) CreateContact(contact *Contact) error { // validate embedded uris in newContact checkList := ControlURIsUniqueness(cb, contact) for uri, check := range checkList { if !check.Ok { return fmt.Errorf("uri <%s> belongs to contact %s", uri, check.OtherContact) } } contactT := cb.IKeyspace.Table("contact", &Contact{}, gocassa.Keys{ PartitionKeys: []string{"user_id", "contact_id"}, }).WithOptions(gocassa.Options{TableName: "contact"}) // need to overwrite default gocassa table naming convention // save contact err := contactT.Set(contact).Run() if err != nil { return fmt.Errorf("[CassandraBackend] CreateContact: %s", err) } isNew := true // create related rows in joined tables go func(*CassandraBackend, *Contact, bool) { err = cb.UpdateRelated(contact, nil, isNew) if err != nil { log.WithError(err).Error("[CassandraBackend] CreateContact : failed to UpdateRelated") } }(cb, contact, isNew) // create related rows in relevant lookup tables go func(*CassandraBackend, *Contact, bool) { err = cb.UpdateLookups(contact, nil, isNew) if err != nil { log.WithError(err).Error("[CassandraBackend] CreateContact : failed to UpdateLookups") } }(cb, contact, isNew) return nil } func (cb *CassandraBackend) RetrieveContact(user_id, contact_id string) (contact *Contact, err error) { // retrieve contact contact = new(Contact).NewEmpty().(*Contact) m := map[string]interface{}{} q := cb.SessionQuery(`SELECT * FROM contact WHERE user_id = ? AND contact_id = ?`, user_id, contact_id) err = q.MapScan(m) if err != nil { return nil, err } if len(m) == 0 { return nil, gocql.ErrNotFound } contact.UnmarshalCQLMap(m) // embed objects from joined tables err = cb.RetrieveRelated(contact) if err != nil { log.WithError(err).Error("[CassandraBackend] RetrieveContact: failed to retrieve related.") } return contact, err } // RetrieveUserContactId returns contactID embedded in user entry // or empty string if error or not found func (cb *CassandraBackend) RetrieveUserContactId(userID string) string { var contactID string err := cb.SessionQuery(`SELECT contact_id FROM user WHERE user_id = ?`, userID).Scan(&contactID) if err != nil { return "" } return contactID } // UpdateContact updates fields into Cassandra // AND updates related lookup tables if needed func (cb *CassandraBackend) UpdateContact(contact, oldContact *Contact, fields map[string]interface{}) error { // validate embedded uris in newContact checkList := ControlURIsUniqueness(cb, contact) for uri, check := range checkList { if !check.Ok { return fmt.Errorf("uri <%s> belongs to contact %s", uri, check.OtherContact) } } //get cassandra's field name for each field to modify cassaFields := map[string]interface{}{} for field, value := range fields { cassaField, err := reflections.GetFieldTag(contact, field, "cql") if err != nil { return fmt.Errorf("[CassandraBackend] UpdateContact failed to find a cql field for object field %s", field) } if cassaField != "-" { cassaFields[cassaField] = value } } contactT := cb.IKeyspace.Table("contact", &Contact{}, gocassa.Keys{ PartitionKeys: []string{"user_id", "contact_id"}, }).WithOptions(gocassa.Options{TableName: "contact"}) // need to overwrite default gocassa table naming convention err := contactT. Where(gocassa.Eq("user_id", contact.UserId.String()), gocassa.Eq("contact_id", contact.ContactId.String())). Update(cassaFields). Run() isNew := false // update related rows in joined tables go func(cb *CassandraBackend, new, old *Contact, isNew bool) { err = cb.UpdateRelated(contact, oldContact, isNew) if err != nil { log.WithError(err).Error("[CassandraBackend] UpdateContact : failed to UpdateRelated") } }(cb, contact, oldContact, isNew) // update related rows in relevant lookup tables go func(cb *CassandraBackend, new, old *Contact, isNew bool) { err = cb.UpdateLookups(contact, oldContact, isNew) if err != nil { log.WithError(err).Error("[CassandraBackend] UpdateContact : failed to UpdateLookups") } }(cb, contact, oldContact, isNew) return nil } // DeleteContact removes Contact from Cassandra // AND removes contactID from related lookup_tables func (cb *CassandraBackend) DeleteContact(contact *Contact) error { // (hard) delete contact. TODO: soft delete err := cb.SessionQuery(`DELETE FROM contact WHERE user_id = ? AND contact_id = ?`, contact.UserId.String(), contact.ContactId.String()).Exec() if err != nil { return err } // delete related rows in joined tables go func(*CassandraBackend, *Contact) { err = cb.DeleteRelated(contact) if err != nil { log.WithError(err).Error("[CassandraBackend] DeleteContact: failed to delete related") } }(cb, contact) // delete related rows in relevant lookup tables go func(*CassandraBackend, *Contact) { err = cb.DeleteLookups(contact) if err != nil { log.WithError(err).Error("[CassandraBackend] DeleteContact: failed to delete lookups") } }(cb, contact) return nil } type UrisCheck struct { Ok bool OtherContact string } type UrisChecklist map[string]UrisCheck // ControlURIsUniqueness checks if all uris embedded in contact belong to this contact only // for earch uri, it returns other contact's id if uri is not available, Ok==true otherwise func ControlURIsUniqueness(cb backends.ContactStorage, contact *Contact) (checkList UrisChecklist) { checkList = UrisChecklist{} for lookup := range contact.GetLookupKeys() { lkp := lookup.(*ContactByContactPoints) contacts, err := cb.LookupContactsByIdentifier(contact.UserId.String(), lkp.Value, lkp.Type) if err != nil { if err.Error() == "not found" { checkList[lkp.Type+":"+lkp.Value] = UrisCheck{Ok: true, OtherContact: ""} } else { checkList[lkp.Type+":"+lkp.Value] = UrisCheck{Ok: false, OtherContact: "error: " + err.Error()} } } for _, c := range contacts { if c != contact.ContactId.String() { checkList[lkp.Type+":"+lkp.Value] = UrisCheck{Ok: false, OtherContact: c} break } } if _, ok := checkList[lkp.Type+":"+lkp.Value]; !ok { checkList[lkp.Type+":"+lkp.Value] = UrisCheck{Ok: true, OtherContact: ""} } } return checkList } func (cb *CassandraBackend) LookupContactsByIdentifier(user_id, address, kind string) (contact_ids []string, err error) { var contact_id string err = cb.SessionQuery(`SELECT contact_id FROM contact_lookup WHERE user_id= ? AND value= ? AND type= ?`, user_id, address, kind).Scan(&contact_id) if err != nil { return nil, err } contact_ids = []string{contact_id} return } // ContactExist exposes a simple API to check if a contact with these uuids exits in db func (cb *CassandraBackend) ContactExists(userId, contactId string) bool { var count int err := cb.SessionQuery(`SELECT count(*) FROM contact WHERE user_id = ? AND contact_id = ?`, userId, contactId).Scan(&count) if err != nil || count == 0 { return false } return true } func (cb *CassandraBackend) ContactsForParticipants(userID string, participants map[string]Participant) error { return ContactsForParticipants(cb.Session, userID, participants) } ================================================ FILE: src/backend/main/go.backends/store/cassandra/contacts_test.go ================================================ package store import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/backendstest" "github.com/satori/go.uuid" "testing" ) func TestCassandraBackend_ControlURIsUniqueness(t *testing.T) { cb := backendstest.GetContactBackend() // test uris not available checkList := ControlURIsUniqueness(cb, &Contact{ Emails: []EmailContact{ { Address: "emma@recovery-caliopen.local", EmailId: UUID(uuid.FromStringOrNil("444d71f6-324c-4733-88a2-77ca28ea6d2d")), IsPrimary: false, Label: "emma@recovery-caliopen.local", Type: "other", }, }, }) if len(checkList) == 0 { t.Error("expected an non empty checklit, got empty") } else { if check, ok := checkList["email:emma@recovery-caliopen.local"]; ok { if check.Ok { t.Error("expected to be unavailable, got OK == true") } else if check.OtherContact != "63ab7904-c416-4f1a-9652-3de82e4fd1f1" { t.Errorf("expected to find another contact with id = 63ab7904-c416-4f1a-9652-3de82e4fd1f1, got %s", check.OtherContact) } } else { t.Error("expected to find key ") } } checkList = ControlURIsUniqueness(cb, &Contact{ Emails: []EmailContact{ { Address: "emma@recovery-caliopen.local", EmailId: UUID(uuid.FromStringOrNil("444d71f6-324c-4733-88a2-77ca28ea6d2d")), IsPrimary: false, Label: "emma@recovery-caliopen.local", Type: "other", }, }, Identities: []SocialIdentity{ { Name: "emmatomme", Type: "twitter", }, }, }) // TODO t.Log(checkList) checkList = ControlURIsUniqueness(cb, &Contact{ Emails: []EmailContact{ { Address: "elvis@able", Type: "other", }, }, Identities: []SocialIdentity{ { Name: "toto", Type: "twitter", }, }, }) // TODO t.Log(checkList) } ================================================ FILE: src/backend/main/go.backends/store/cassandra/credentials.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package store import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/gocassa/gocassa" ) func (cb *CassandraBackend) CreateCredentials(userIdentity *UserIdentity, cred Credentials) error { if cb.UseVault { return cb.Vault.CreateCredentials(userIdentity, cred) } //(re)embed credentials into UserIdentity that has already been created (*userIdentity).Credentials = &cred return cb.UpdateUserIdentity(userIdentity, map[string]interface{}{ "Credentials": cred, }) } func (cb *CassandraBackend) RetrieveCredentials(userId, identityId string) (cred Credentials, err error) { if cb.UseVault { return cb.Vault.RetrieveCredentials(userId, identityId) } err = cb.SessionQuery(`SELECT credentials FROM user_identity WHERE user_id = ? AND identity_id = ?`, userId, identityId).Scan(&cred) return } func (cb *CassandraBackend) UpdateCredentials(userId, identityId string, cred Credentials) error { if cb.UseVault { return cb.Vault.UpdateCredentials(userId, identityId, cred, false) } // check if identity exists before executing UPDATE because `IF EXISTS` statement not supported by scylladb as of february 2019 if cb.SessionQuery(`SELECT user_id FROM user_identity WHERE user_id = ? AND identity_id = ?`, userId, identityId).Iter().NumRows() == 0 { return errors.New("not found") } userIdentityTable := cb.IKeyspace.Table("user_identity", &UserIdentity{}, gocassa.Keys{ PartitionKeys: []string{"user_id", "identity_id"}, }).WithOptions(gocassa.Options{TableName: "user_identity"}) return userIdentityTable.Where(gocassa.Eq("user_id", userId), gocassa.Eq("identity_id", identityId)). Update(map[string]interface{}{ "credentials": cred, }).Run() } func (cb *CassandraBackend) DeleteCredentials(userId, identityId string) error { if cb.UseVault { return cb.Vault.DeleteCredentials(userId, identityId) } return cb.UpdateCredentials(userId, identityId, Credentials{}) } ================================================ FILE: src/backend/main/go.backends/store/cassandra/devices.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package store import ( "errors" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "github.com/gocassa/gocassa" "gopkg.in/oleiade/reflections.v1" ) func (cb *CassandraBackend) CreateDevice(device *Device) error { deviceT := cb.IKeyspace.Table("device", &Device{}, gocassa.Keys{ PartitionKeys: []string{"user_id", "device_id"}, }).WithOptions(gocassa.Options{TableName: "device"}) //save device err := deviceT.Set(device).Run() if err != nil { return fmt.Errorf("[CassandraBackend] CreateContact: %s", err) } isNew := true // create related rows in joinde tables (if any) go func(*CassandraBackend, *Device, bool) { err = cb.UpdateRelated(device, nil, isNew) if err != nil { log.WithError(err).Error("[CassandraBackend] CreateDevice : failed to UpdateRelated") } }(cb, device, isNew) /*** NO LOOKUPS for now, code below will be uncommented if needed ***/ // create related rows in relevant lookup tables (if any) /* go func(*CassandraBackend, *Device, bool) { err = cb.UpdateLookups(device, nil, isNew) if err != nil { log.WithError(err).Error("[CassandraBackend] CreateDevice : failed to UpdateLookups") } }(cb, device, isNew) */ return nil } // retrieve devices belonging to user_id func (cb *CassandraBackend) RetrieveDevices(userId string) (devices []Device, err error) { all_devices, err := cb.SessionQuery(`SELECT * FROM device WHERE user_id = ?`, userId).Iter().SliceMap() if err != nil { return } if len(all_devices) == 0 { err = errors.New("devices not found") return } for _, device := range all_devices { d := new(Device).NewEmpty().(*Device) d.UnmarshalCQLMap(device) // embed objects from joined tables err = cb.RetrieveRelated(d) if err != nil { log.WithError(err).Error("[CassandraBackend] RetrieveDevice: failed to retrieve related.") } else { devices = append(devices, *d) } } return } func (cb *CassandraBackend) RetrieveDevice(userId, deviceId string) (device *Device, err error) { device = new(Device).NewEmpty().(*Device) d := map[string]interface{}{} q := cb.SessionQuery(`SELECT * FROM device WHERE user_id = ? AND device_id = ?`, userId, deviceId) err = q.MapScan(d) if err != nil { return nil, err } if len(d) == 0 { err = errors.New("not found") return nil, err } device.UnmarshalCQLMap(d) // embed objects from joined tables err = cb.RetrieveRelated(device) if err != nil { log.WithError(err).Error("[CassandraBackend] RetrieveDevice: failed to retrieve related.") } return device, nil } func (cb *CassandraBackend) UpdateDevice(device, oldDevice *Device, fields map[string]interface{}) error { //get cassandra's field name for each field to modify cassaFields := map[string]interface{}{} for field, value := range fields { cassaField, err := reflections.GetFieldTag(device, field, "cql") if err != nil { return fmt.Errorf("[CassandraBackend] UpdateDevice failed to find a cql field for object field %s", field) } if cassaField != "-" { cassaFields[cassaField] = value } } deviceT := cb.IKeyspace.Table("device", &Device{}, gocassa.Keys{ PartitionKeys: []string{"user_id", "device_id"}, }).WithOptions(gocassa.Options{TableName: "device"}) // need to overwrite default gocassa table naming convention err := deviceT. Where(gocassa.Eq("user_id", device.UserId.String()), gocassa.Eq("device_id", device.DeviceId.String())). Update(cassaFields). Run() isNew := false // update related rows in joined tables go func(cb *CassandraBackend, new, old *Device, isNew bool) { err = cb.UpdateRelated(device, oldDevice, isNew) if err != nil { log.WithError(err).Error("[CassandraBackend] Updatedevice : failed to UpdateRelated") } }(cb, device, oldDevice, isNew) // update related rows in relevant lookup tables /*** no lookups for now go func(cb *CassandraBackend, new, old *Device, isNew bool) { err = cb.UpdateLookups(device, oldDevice, isNew) if err != nil { log.WithError(err).Error("[CassandraBackend] Updatedevice : failed to UpdateLookups") } }(cb, device, oldDevice, isNew) ***/ return nil } func (cb *CassandraBackend) DeleteDevice(device *Device) error { // (hard) delete device. TODO: soft delete err := cb.SessionQuery(`DELETE FROM device WHERE user_id = ? AND device_id = ?`, device.UserId.String(), device.DeviceId.String()).Exec() if err != nil { return err } // delete related rows in joined tables go func(*CassandraBackend, *Device) { err = cb.DeleteRelated(device) if err != nil { log.WithError(err).Error("[CassandraBackend] DeleteDevice: failed to delete related") } }(cb, device) // delete related rows in relevant lookup tables /*** no lookups for now go func(*CassandraBackend, *Device) { err = cb.DeleteLookups(device) if err != nil { log.WithError(err).Error("[CassandraBackend] Deletedevice: failed to delete lookups") } }(cb, device) ***/ return nil } ================================================ FILE: src/backend/main/go.backends/store/cassandra/discussions.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package store import ( "errors" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/gocassa/gocassa" "time" ) func (cb *CassandraBackend) GetUserLookupHashes(userId UUID, kind, key string) (hashes []ParticipantHash, err error) { var rawHashes []map[string]interface{} if key != "" { rawHashes, err = cb.SessionQuery(`SELECT * from participant_hash WHERE user_id = ? AND kind = ? AND key = ?`, userId, kind, key).Iter().SliceMap() } else { rawHashes, err = cb.SessionQuery(`SELECT * from participant_hash WHERE user_id = ? AND kind = ?`, userId, kind).Iter().SliceMap() } if err != nil { return } if len(rawHashes) == 0 { err = errors.New("not found") return } for _, hash := range rawHashes { h := new(ParticipantHash) h.UnmarshalCQLMap(hash) hashes = append(hashes, *h) } return } func (cb *CassandraBackend) RetrieveParticipantHash(userId UUID, kind, hash string) (lookup ParticipantHash, err error) { m := map[string]interface{}{} q := cb.SessionQuery(`SELECT * from participant_hash WHERE user_id = ? AND kind = ? AND key = ?`, userId, kind, hash) err = q.MapScan(m) if err != nil { return } lookup.UnmarshalCQLMap(m) return } func (cb *CassandraBackend) CreateParticipantHash(lookup *ParticipantHash) error { lookupT := cb.IKeyspace.Table("participant_hash", &ParticipantHash{}, gocassa.Keys{ PartitionKeys: []string{"user_id", "kind", "key", "value"}, }).WithOptions(gocassa.Options{TableName: "participant_hash"}) // need to overwrite default gocassa table naming convention // save lookup err := lookupT.Set(lookup).Run() if err != nil { return fmt.Errorf("[CassandraBackend] CreateParticipantHash: %s", err) } return nil } // UpsertDiscussionLookups ensures that relevant entries are present in both HashLookup and ParticipantHash tables // for the provided participants, taking into account user's contacts addresses func (cb *CassandraBackend) UpsertDiscussionLookups(userId UUID, participants []Participant) error { hash, components, err := HashFromParticipantsUris(participants) if err != nil { return err } return cb.CreateLookupsFromUris(userId, hash, components) } // CreateLookupsFromUris resolves uris to contact to build participants' set // then computes participants_hash // then creates bijection in HashLookup and ParticipantLookup tables: // uris<->uris_hash // uris_hash<->participants_hash func (cb *CassandraBackend) CreateLookupsFromUris(userId UUID, uriHash string, uris []string) error { now := time.Now() for _, uri := range uris { // store uri->uris_hash e := cb.CreateHashLookup(HashLookup{ UserId: userId, Uri: uri, Hash: uriHash, HashComponents: uris, DateInsert: now, }) if e != nil { return e } } // pick one uri to trigger participant_hash table update if len(uris) > 0 { return UpdateURIWithContact(cb.Session, userId.String(), uris[0]) } return nil } ================================================ FILE: src/backend/main/go.backends/store/cassandra/discussions_test.go ================================================ package store ================================================ FILE: src/backend/main/go.backends/store/cassandra/emails.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package store import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/satori/go.uuid" ) // GetUsersForLocalMailRecipients is part of LDABackend interface implementation // return a list of tuples ([user_id, identity_id]) of **local** users found for the given email addresses func (cb *CassandraBackend) GetUsersForLocalMailRecipients(rcpts []string) (userIds [][]UUID, err error) { userIds = [][]UUID{} for _, rcpt := range rcpts { identities, err := cb.LookupIdentityByIdentifier(rcpt) if err == nil { for _, identity := range identities { if cb.IsLocalIdentity(identity[0], identity[1]) { userIds = append(userIds, []UUID{UUID(uuid.FromStringOrNil(identity[0])), UUID(uuid.FromStringOrNil(identity[1]))}) } } } } return } ================================================ FILE: src/backend/main/go.backends/store/cassandra/identities.go ================================================ // Copyleft (ɔ) 2018 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package store import ( "errors" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "github.com/gocassa/gocassa" "github.com/gocql/gocql" "gopkg.in/oleiade/reflections.v1" "strings" "time" ) func (cb *CassandraBackend) RetrieveLocalsIdentities(userId string) (identities []UserIdentity, err error) { var count int iter := cb.SessionQuery(`SELECT identity_id FROM identity_type_lookup WHERE type = ? AND user_id = ?`, LocalIdentity, userId).Iter() for { var identityID gocql.UUID if !iter.Scan(&identityID) { break } count++ i := make(map[string]interface{}) cb.SessionQuery(`SELECT * FROM user_identity WHERE user_id = ? AND identity_id = ?`, userId, identityID).MapScan(i) identity := new(UserIdentity) identity.UnmarshalCQLMap(i) identities = append(identities, *identity) } if count == 0 { err = errors.New("not found") return } return } func (cb *CassandraBackend) CreateUserIdentity(userIdentity *UserIdentity) CaliopenError { userIdentity.Identifier = strings.ToLower(userIdentity.Identifier) userIdentityTable := cb.IKeyspace.Table("user_identity", &UserIdentity{}, gocassa.Keys{ PartitionKeys: []string{"user_id", "identity_id"}, }).WithOptions(gocassa.Options{TableName: "user_identity"}) // check if user identity already exist var count int err := cb.SessionQuery(`SELECT count(*) from user_identity WHERE user_id = ? AND identity_id = ?`, userIdentity.UserId, userIdentity.Id).Scan(&count) if err != nil { log.WithError(err).Errorf("[CassandraBackend] SELECT fails for user_id=%s, identity_id=%s", userIdentity.UserId.String(), userIdentity.Id.String()) return WrapCaliopenErrf(err, DbCaliopenErr, "[CassandraBackend] CreateUserIdentity fails : <%s>", err.Error()) } if count != 0 { return NewCaliopenErrf(ForbiddenCaliopenErr, "[CassandraBackend] CreateUserIdentity error : user identity <%s> already exist for user <%s>", userIdentity.Id, userIdentity.UserId.String()) } // remove credentials from struct cred := userIdentity.Credentials (*userIdentity).Credentials = nil // create user identity err = userIdentityTable.Set(userIdentity).Run() if err != nil { log.WithError(err).Errorf("[CassandraBackend] set fails for %+v", userIdentity) return WrapCaliopenErrf(err, DbCaliopenErr, "[CassandraBackend] CreateUserIdentity fails : %s", err.Error()) } // update lookup tables err = cb.UpdateLookups(userIdentity, nil, true) if err != nil { log.WithError(err).Warnf("[CassandraBackend] UpdateLookups error for UserIdentity %s (user %s)", userIdentity.Id.String(), userIdentity.UserId.String()) } // create credentials apart if cred != nil { err = cb.CreateCredentials(userIdentity, *cred) if err != nil { log.WithError(err).Errorf("[CassandraBackend] create credentials failed for %+v", *cred) return WrapCaliopenErr(err, DbCaliopenErr, "[CassandraBackend) CreateUserIdentity failed to createCredentials") } } return nil } func (cb *CassandraBackend) RetrieveUserIdentity(userId, identityId string, withCredentials bool) (userIdentity *UserIdentity, err error) { userIdentity = new(UserIdentity) m := map[string]interface{}{} q := cb.SessionQuery(`SELECT * FROM user_identity WHERE user_id = ? AND identity_id = ?`, userId, identityId) err = q.MapScan(m) if err != nil { return nil, err } userIdentity.UnmarshalCQLMap(m) if withCredentials && userIdentity.Type != LocalIdentity { cred, err := cb.RetrieveCredentials(userId, identityId) if err != nil { return nil, err } userIdentity.Credentials = &cred } else { // discard credentials userIdentity.Credentials = nil } return } func (cb *CassandraBackend) UpdateUserIdentity(userIdentity *UserIdentity, fields map[string]interface{}) (err error) { userIdentity.Identifier = strings.ToLower(userIdentity.Identifier) // check if identity exists before executing UPDATE because `IF EXISTS` statement not supported by scylladb as of february 2019 if cb.SessionQuery(`SELECT user_id FROM user_identity WHERE user_id = ? AND identity_id = ?`, userIdentity.UserId.String(), userIdentity.Id.String()).Iter().NumRows() == 0 { return errors.New("not found") } //remove Credentials from userIdentity and process this special property apart if cred, ok := fields["Credentials"].(*Credentials); ok && cred != nil { (*userIdentity).Credentials = nil delete(fields, "Credentials") err = cb.UpdateCredentials(userIdentity.UserId.String(), userIdentity.Id.String(), *cred) if err != nil { log.WithError(err).Warn("[CassandraBackend] UpdateUserIdentity failed to update credentials") } } if identifier, ok := fields["identifier"]; ok { fields["identifier"] = strings.ToLower(identifier.(string)) } if len(fields) > 0 { //get cassandra's field name for each field to modify cassaFields := map[string]interface{}{} for field, value := range fields { cassaField, err := reflections.GetFieldTag(userIdentity, field, "cql") if err != nil { return fmt.Errorf("[CassandraBackend] UpdateUserIdentity failed to find a cql field for object field %s", field) } if cassaField != "-" { cassaFields[cassaField] = value } } userIdentityTable := cb.IKeyspace.Table("user_identity", &UserIdentity{}, gocassa.Keys{ PartitionKeys: []string{"user_id", "identity_id"}, }).WithOptions(gocassa.Options{TableName: "user_identity"}) err = userIdentityTable.Where(gocassa.Eq("user_id", userIdentity.UserId.String()), gocassa.Eq("identity_id", userIdentity.Id.String())). Update(cassaFields).Run() if err != nil { log.WithError(err).Errorf("[CassandraBackend]UpdateUserIdentity failed to UPDATE with fields %+v", cassaFields) return err } // update lookup tables return cb.UpdateLookups(userIdentity, nil, false) } return err } // UpdateRemoteInfos is a convenient way to quickly update infos map without the need of an already created UserIdentity object func (cb *CassandraBackend) UpdateRemoteInfosMap(userId, remoteId string, infos map[string]string) error { // check if identity exists before executing UPDATE because `IF EXISTS` statement not supported by scylladb as of february 2019 if cb.SessionQuery(`SELECT user_id FROM user_identity WHERE user_id = ? AND identity_id = ?`, userId, remoteId).Iter().NumRows() == 0 { return errors.New("not found") } userIdentityTable := cb.IKeyspace.Table("user_identity", &UserIdentity{}, gocassa.Keys{ PartitionKeys: []string{"user_id", "identity_id"}, }).WithOptions(gocassa.Options{TableName: "user_identity"}) return userIdentityTable.Where(gocassa.Eq("user_id", userId), gocassa.Eq("identity_id", remoteId)). Update(map[string]interface{}{ "infos": infos, }).Run() } // RetrieveRemoteInfos is a convenient way to quickly retrieve infos map without the need of an already created UserIdentity object func (cb *CassandraBackend) RetrieveRemoteInfosMap(userId, remoteId string) (infos map[string]string, err error) { m := map[string]interface{}{} infos = map[string]string{} q := cb.SessionQuery(`SELECT infos FROM user_identity WHERE user_id = ? AND identity_id = ?`, userId, remoteId) err = q.MapScan(m) if err != nil { return nil, err } for k, v := range m["infos"].(map[string]string) { infos[k] = v } return } func (cb *CassandraBackend) RetrieveRemoteIdentities(userId string, withCredentials bool) (userIdentities []*UserIdentity, err error) { var count int iter := cb.SessionQuery(`SELECT identity_id FROM identity_type_lookup WHERE type = ? AND user_id = ?`, RemoteIdentity, userId).Iter() for { var identityID gocql.UUID if !iter.Scan(&identityID) { break } count++ i := make(map[string]interface{}) e := cb.SessionQuery(`SELECT * FROM user_identity WHERE user_id = ? AND identity_id = ?`, userId, identityID).MapScan(i) if e != nil { log.Warnf("[CassandraBackend] RetrieveRemoteIdentities failed on inconsistency between identity_type_lookup and user_identity for user %s and identity %s", userId, identityID) continue } identity := new(UserIdentity) identity.UnmarshalCQLMap(i) if withCredentials { cred, err := cb.RetrieveCredentials(userId, identity.Id.String()) if err != nil { // return user identity even if credentials retrieval failed cred = Credentials{} } identity.Credentials = &cred } else { // discard credentials identity.Credentials = nil } userIdentities = append(userIdentities, identity) } if count == 0 { err = errors.New("not found") return } return } // RetrieveAllRemotes returns a chan to range over all remote identities found in db func (cb *CassandraBackend) RetrieveAllRemotes(withCredentials bool) (<-chan *UserIdentity, error) { ch := make(chan *UserIdentity) go func(cb *CassandraBackend, ch chan *UserIdentity) { iter := cb.SessionQuery(`SELECT user_id, identity_id from identity_type_lookup WHERE type = ?`, RemoteIdentity).Iter() for { var userId, identityId gocql.UUID if !iter.Scan(&userId, &identityId) { break } userIdentity, err := cb.RetrieveUserIdentity(userId.String(), identityId.String(), withCredentials) if err != nil { log.WithError(err).Warnf("[CassandraBackend]RetrieveAllRemotes fails to retrieve identity for user %s and identity_id %s", userId.String(), identityId.String()) continue } if withCredentials { cred, err := cb.RetrieveCredentials(userIdentity.UserId.String(), userIdentity.Id.String()) if err != nil { userIdentity.Credentials = &cred } else { userIdentity.Credentials = nil } } else { userIdentity.Credentials = nil } select { case ch <- userIdentity: case <-time.After(cb.Timeout): log.Warn("[RetrieveAllRemote] write timeout on chan") } } iter.Close() close(ch) }(cb, ch) return ch, nil } func (cb *CassandraBackend) DeleteUserIdentity(userIdentity *UserIdentity) error { if cb.UseVault { err := cb.Vault.DeleteCredentials(userIdentity.UserId.String(), userIdentity.Id.String()) if err != nil { log.WithError(err).Warn("[CassandraBackend] DeleteUserIdentity failed to delete credentials in vault") } } // delete related rows in relevant lookup tables err := cb.DeleteLookups(userIdentity) if err != nil { log.WithError(err).Error("[CassandraBackend] DeleteUserIdentity: failed to delete lookups") } return cb.SessionQuery(`DELETE FROM user_identity WHERE user_id = ? AND identity_id = ?`, userIdentity.UserId, userIdentity.Id).Exec() } // LookupIdentityByIdentifier retrieve one or more identity_id depending on given parameters : // an identifier (mandatory) // other params could be protocol string, user_id string // returns an array of [user_id, identity_id] func (cb *CassandraBackend) LookupIdentityByIdentifier(identifier string, params ...string) (identities [][2]string, err error) { if identifier == "" { err = errors.New("identifier is mandatory") return } identifier = strings.ToLower(identifier) var rows []map[string]interface{} switch len(params) { case 0: rows, err = cb.SessionQuery(`SELECT * from identity_lookup WHERE identifier = ?`, identifier).Iter().SliceMap() if err != nil { return } case 1: rows, err = cb.SessionQuery(`SELECT * from identity_lookup WHERE identifier = ? AND protocol = ?`, identifier, params[0]).Iter().SliceMap() if err != nil { return } case 2: rows, err = cb.SessionQuery(`SELECT * from identity_lookup WHERE identifier = ? AND protocol = ? AND user_id = ?`, identifier, params[0], params[1]).Iter().SliceMap() if err != nil { return } default: err = errors.New("too many params provided") return } for _, row := range rows { identities = append(identities, [2]string{ row["user_id"].(gocql.UUID).String(), row["identity_id"].(gocql.UUID).String(), }) } return } // LookupIdentityByType retrieve one or more identity_id depending on given parameters : // a type (mandatory) // a user_id (optional) // returns an array of [user_id, identity_id] func (cb *CassandraBackend) LookupIdentityByType(identityType string, user_id ...string) (identities [][2]string, err error) { if identityType == "" { err = errors.New("identity type is mandatory") return } var rows []map[string]interface{} switch len(user_id) { case 0: rows, err = cb.SessionQuery(`SELECT * from identity_type_lookup WHERE type = ?`, identityType).Iter().SliceMap() if err != nil { return } case 1: rows, err = cb.SessionQuery(`SELECT * from identity_type_lookup WHERE type = ? AND user_id = ?`, identityType, user_id[0]).Iter().SliceMap() if err != nil { return } default: err = errors.New("too many user_id provided") return } for _, row := range rows { identities = append(identities, [2]string{ row["user_id"].(gocql.UUID).String(), row["identity_id"].(gocql.UUID).String(), }) } return } // IsLocalIdentity is helper to make a lookup query in cassandra and check if a UserIdentity is "local" // return true only if identity has been found and is local func (cb *CassandraBackend) IsLocalIdentity(userId, identityId string) bool { var identityType string err := cb.SessionQuery(`SELECT type FROM user_identity WHERE user_id = ? AND identity_id = ?`, userId, identityId).Scan(&identityType) if err != nil || identityType != LocalIdentity { return false } return true } // IsRemoteIdentity is helper to make a lookup query in cassandra and check if a UserIdentity is "local" // return true only if identity has been found and is local func (cb *CassandraBackend) IsRemoteIdentity(userId, identityId string) bool { var identityType string err := cb.SessionQuery(`SELECT type FROM user_identity WHERE user_id = ? AND identity_id = ?`, userId, identityId).Scan(&identityType) if err != nil || identityType != RemoteIdentity { return false } return true } // TimestampRemoteLastCheck writes timestamp to user_identity.last_check property. // If no time is provided defaults to time.Now() func (cb *CassandraBackend) TimestampRemoteLastCheck(userId, remoteId string, t ...time.Time) error { // check if identity exists before executing UPDATE because `IF EXISTS` statement not supported by scylladb as of february 2019 if cb.SessionQuery(`SELECT user_id FROM user_identity WHERE user_id = ? AND identity_id = ?`, userId, remoteId).Iter().NumRows() == 0 { return errors.New("not found") } var timestamp time.Time if len(t) < 1 { timestamp = time.Now() } else { timestamp = t[0] } return cb.SessionQuery(`UPDATE user_identity SET last_check = ? WHERE user_id = ? AND identity_id = ?`, timestamp, userId, remoteId).Exec() } ================================================ FILE: src/backend/main/go.backends/store/cassandra/keys.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package store import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/gocassa/gocassa" "gopkg.in/oleiade/reflections.v1" ) func (cb *CassandraBackend) CreatePGPPubKey(pubkey *PublicKey) CaliopenError { // write complete statement because gocassa failed to retrieve tag "use" and write quotes to cassandra err := cb.SessionQuery(`UPDATE public_key SET alg = ?, crv = ?, date_insert = ?, date_update = ?, emails = ?, expire_date = ?, fingerprint = ?, key = ?, kty = ?, label = ?, resource_type = ?, size = ?, "use" = ?, x = ?, y = ? WHERE user_id = ? AND resource_id = ? AND key_id = ?`, pubkey.Algorithm, pubkey.Curve, pubkey.DateInsert, pubkey.DateUpdate, pubkey.Emails, pubkey.ExpireDate, pubkey.Fingerprint, pubkey.Key, pubkey.KeyType, pubkey.Label, pubkey.ResourceType, pubkey.Size, pubkey.Use, pubkey.X, pubkey.Y, pubkey.UserId, pubkey.ResourceId, pubkey.KeyId).Exec() if err != nil { return NewCaliopenErrf(DbCaliopenErr, "[CassandraBackend]CreatePGPPubKey db error : %s", err.Error()) } return nil } func (cb *CassandraBackend) RetrieveContactPubKeys(userId, contactId string) (keys PublicKeys, err CaliopenError) { ks, e := cb.SessionQuery(`SELECT * FROM public_key WHERE user_id = ? AND resource_id = ?`, userId, contactId).Iter().SliceMap() if e != nil { return nil, WrapCaliopenErrf(e, DbCaliopenErr, "[CassandraBackend]RetrieveContactPubKeys failed") } for _, k := range ks { pubkey := new(PublicKey) pubkey.UnmarshalCQLMap(k) keys = append(keys, *pubkey) } return } func (cb *CassandraBackend) RetrievePubKey(userId, resourceId, keyId string) (pubkey *PublicKey, err CaliopenError) { result := map[string]interface{}{} e := cb.SessionQuery(`SELECT * FROM public_key WHERE user_id = ? AND resource_id = ? AND key_id = ?`, userId, resourceId, keyId).MapScan(result) if e != nil { if e.Error() == "not found" { err = WrapCaliopenErr(NewCaliopenErr(NotFoundCaliopenErr, "not found"), DbCaliopenErr, "[CassandraBackend]RetrievePubKey not found in db") } else { err = NewCaliopenErrf(DbCaliopenErr, "[CassandraBackend]RetrievePubKey returned error from cassandra : %s", e.Error()) } return } pubkey = new(PublicKey) pubkey.UnmarshalCQLMap(result) return } func (cb *CassandraBackend) UpdatePubKey(newPubKey, oldPubKey *PublicKey, fields map[string]interface{}) CaliopenError { //get cassandra's field name for each field to modify cassaFields := map[string]interface{}{} for field, value := range fields { cassaField, err := reflections.GetFieldTag(newPubKey, field, "cql") if err != nil { return NewCaliopenErrf(FailDependencyCaliopenErr, "[CassandraBackend]UpdatePubKey failed to find a cql field for object field %s", field) } if cassaField != "-" { cassaFields[cassaField] = value } } keyT := cb.IKeyspace.Table("public_key", &PublicKey{}, gocassa.Keys{ PartitionKeys: []string{"user_id", "resource_id", "key_id"}, }).WithOptions(gocassa.Options{TableName: "public_key"}) // need to overwrite default gocassa table naming convention err := keyT. Where(gocassa.Eq("user_id", newPubKey.UserId.String()), gocassa.Eq("resource_id", newPubKey.ResourceId.String()), gocassa.Eq("key_id", newPubKey.KeyId.String())). Update(cassaFields). Run() if err != nil { return NewCaliopenErrf(DbCaliopenErr, "[CassandraBackend]UpdatePubKey failed to call store with cassandra error : %s", err.Error()) } return nil } func (cb *CassandraBackend) DeletePubKey(pubkey *PublicKey) CaliopenError { e := cb.SessionQuery(`DELETE FROM public_key WHERE user_id = ? AND resource_id = ? AND key_id = ?`, pubkey.UserId, pubkey.ResourceId, pubkey.KeyId).Exec() if e != nil { return NewCaliopenErrf(DbCaliopenErr, "[CassandraBackend]DeletePubKey returned err from cassandra : %s", e.Error()) } return nil } ================================================ FILE: src/backend/main/go.backends/store/cassandra/lookups.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package store import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" ) // UpdateLookups ensure that tables lookup related to are up to date in db, // it updates values with `new` state and delete lookups that have been removed func (cb *CassandraBackend) UpdateLookups(new, old HasLookup, isNew bool) error { var err error if cb.Session.Closed() { err = (*cb).initialize(cb.CassandraConfig) if err != nil { log.WithError(err).Warn("[CassandraBackend] UpdateLookups failed to re-init a gocql Session") } } // update db with current values for _, lookup := range new.GetLookupsTables() { if updateFunc := lookup.UpdateLookups(new, old); updateFunc != nil { err = updateFunc(cb.Session) if err != nil { return err } } } return nil } // DeleteLookups ensure that tables related (joined) to obj are up to date in db, // ie related objects are deleted accordingly. func (cb *CassandraBackend) DeleteLookups(obj HasLookup) error { var err error if cb.Session.Closed() { err = (*cb).initialize(cb.CassandraConfig) if err != nil { log.WithError(err).Warn("[CassandraBackend] DeleteLookups failed to re-init a gocql Session") } } for _, lookup := range obj.GetLookupsTables() { if cleanup := lookup.CleanupLookups(obj); cleanup != nil { err = cleanup(cb.Session) if err != nil { return err } } } return nil } ================================================ FILE: src/backend/main/go.backends/store/cassandra/messages.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package store import ( "errors" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/Sirupsen/logrus" "github.com/gocassa/gocassa" "github.com/gocql/gocql" "gopkg.in/oleiade/reflections.v1" ) func (cb *CassandraBackend) CreateMessage(msg *Message) error { messageT := cb.IKeyspace.Table("message", &Message{}, gocassa.Keys{ PartitionKeys: []string{"user_id", "message_id"}, }).WithOptions(gocassa.Options{TableName: "message"}) // need to overwrite default gocassa table naming convention return messageT.Set(msg).Run() } func (cb *CassandraBackend) RetrieveMessage(user_id, msg_id string) (msg *Message, err error) { msg = new(Message).NewEmpty().(*Message) // correctly initialize nested values m := map[string]interface{}{} err = cb.SessionQuery(`SELECT * FROM message WHERE user_id = ? and message_id = ?`, user_id, msg_id).MapScan(m) if err != nil { return nil, err } msg.UnmarshalCQLMap(m) // embed up-to-date participants' details MessagesParticipantsDetails(cb.GetSession(), []Message{*msg}) return msg, err } // update given fields for a message in db func (cb *CassandraBackend) UpdateMessage(msg *Message, fields map[string]interface{}) error { //get cassandra's field name for each field to modify cassaFields := map[string]interface{}{} for field, value := range fields { cassaField, err := reflections.GetFieldTag(msg, field, "cql") if err != nil { return fmt.Errorf("[CassandraBackend] UpdateMessage failed to find a cql field for object field %s", field) } cassaFields[cassaField] = value } messageT := cb.IKeyspace.Table("message", &Message{}, gocassa.Keys{ PartitionKeys: []string{"user_id", "message_id"}, }).WithOptions(gocassa.Options{TableName: "message"}) // need to overwrite default gocassa table naming convention err := messageT. Where(gocassa.Eq("user_id", msg.User_id.String()), gocassa.Eq("message_id", msg.Message_id.String())). Update(cassaFields). Run() if externalRefs, ok := fields["External_references"].(ExternalReferences); ok { // need to update lookup table err := cb.SessionQuery(`INSERT INTO message_external_ref_lookup (user_id,external_msg_id,identity_id,message_id) VALUES (?,?,?,?)`, msg.User_id, externalRefs.Message_id, msg.UserIdentities[0], msg.Message_id).Exec() if err != nil { logrus.WithError(err).Errorf("UpdateMessage failed to update external ref lookup for user %s, message id %s, user identity %s, external ref %s", msg.User_id, msg.Message_id, msg.UserIdentities[0], externalRefs.Message_id) } } return err } func (cb *CassandraBackend) DeleteMessage(msg *Message) error { return errors.New("[CassandraBackend] DeleteMessage not yet implemented") } func (cb *CassandraBackend) SetMessageUnread(user_id, message_id string, status bool) (err error) { q := cb.SessionQuery(`UPDATE message SET is_unread= ? WHERE message_id = ? AND user_id = ?`, status, message_id, user_id) return q.Exec() } // SeekMessageByExternalRef return first message found in cassandra's message_external_ref_lookup table, if any. // if identityID param is an empty string, `identity_id` key will be ignored in cql request func (cb *CassandraBackend) SeekMessageByExternalRef(userID, externalMessageID, identityID string) (messageID UUID, err error) { result := map[string]interface{}{} if identityID == "" { err = cb.SessionQuery(`SELECT message_id FROM message_external_ref_lookup WHERE user_id = ? AND external_msg_id = ? LIMIT 1"`, userID, externalMessageID).MapScan(result) } else { err = cb.SessionQuery(`SELECT message_id FROM message_external_ref_lookup WHERE user_id = ? AND external_msg_id = ? AND identity_id = ?`, userID, externalMessageID, identityID).MapScan(result) } if err != nil || result["message_id"] == nil { return EmptyUUID, err } return UUID(result["message_id"].(gocql.UUID)), err } ================================================ FILE: src/backend/main/go.backends/store/cassandra/notifications.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package store import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "github.com/gocassa/gocassa" "github.com/gocql/gocql" "strings" "time" ) func (cb *CassandraBackend) PutNotificationInQueue(notif *Notification) error { var ttl NotificationTTL // TODO: find a way to avoid retrieving duration from cassandra for each Put err := cb.SessionQuery(`SELECT * FROM notification_ttl WHERE ttl_code = ?`, notif.TTLcode).Scan(&ttl.TTLcode, &ttl.Description, &ttl.TTLduration) if err != nil { log.WithError(err).Error("[CassandraBackend]PutNotificationInQueue failed to retrieve ttl") return err } notifT := cb.IKeyspace.Table("notification", &NotificationModel{}, gocassa.Keys{ PartitionKeys: []string{"user_id", "notif_id"}, }).WithOptions(gocassa.Options{TableName: "notification"}) n := NotificationModel{ Body: notif.Body, Emitter: notif.Emitter, NotifId: notif.NotifId.String(), Reference: notif.Reference, Type: notif.Type, UserId: notif.User.UserId.String(), } return notifT.Set(&n).WithOptions(gocassa.Options{TTL: time.Duration(ttl.TTLduration) * time.Second}).Run() } func (cb *CassandraBackend) NotificationsByTime(userId string, from, to time.Time) ([]Notification, error) { var query_builder strings.Builder values := []interface{}{userId} notifs := []Notification{} query_builder.WriteString(`SELECT * FROM notification WHERE user_id = ?`) if !from.IsZero() { query_builder.WriteString(` AND notif_id > minTimeuuid(?)`) values = append(values, from) } if !to.IsZero() { query_builder.WriteString(` AND notif_id < maxTimeuuid(?)`) values = append(values, to) } notifs_found, err := cb.SessionQuery(query_builder.String(), values...).Iter().SliceMap() if err != nil { return notifs, err } if len(notifs_found) == 0 { return []Notification{}, errors.New("notifications not found") } for _, notif := range notifs_found { n := new(Notification) n.UnmarshalCQLMap(notif) notifs = append(notifs, *n) } return notifs, nil } func (cb *CassandraBackend) NotificationsByID(userID, from, to string) ([]Notification, error) { var query_builder strings.Builder values := []interface{}{userID} notifs := []Notification{} query_builder.WriteString(`SELECT * FROM notification WHERE user_id = ?`) if from != "" { query_builder.WriteString(` AND notif_id > ?`) values = append(values, from) } if to != "" { query_builder.WriteString(` AND notif_id < ?`) values = append(values, to) } notifs_found, err := cb.SessionQuery(query_builder.String(), values...).Iter().SliceMap() if err != nil { return notifs, err } if len(notifs_found) == 0 { return []Notification{}, errors.New("notifications not found") } for _, notif := range notifs_found { n := new(Notification) n.UnmarshalCQLMap(notif) notifs = append(notifs, *n) } return notifs, nil } func (cb *CassandraBackend) DeleteNotifications(userId string, until time.Time) error { var query_builder strings.Builder values := []interface{}{userId} /*** **** as of 2018 may 16th, production server is running cassandra 2.4.1, **** which is not able to do a range DELETE. query_builder.WriteString(`DELETE FROM notification WHERE user_id = ?`) if !until.IsZero() { query_builder.WriteString(` AND notif_id > minTimeuuid(0) AND notif_id < maxTimeuuid(?)`) values = append(values, until) } return cb.Session.Query(query_builder.String(), values...).Exec() */ query_builder.WriteString(`SELECT notif_id FROM notification WHERE user_id = ?`) if !until.IsZero() { query_builder.WriteString(` AND notif_id > minTimeuuid(0) AND notif_id < maxTimeuuid(?)`) values = append(values, until) } var notifId gocql.UUID iter := cb.SessionQuery(query_builder.String(), values...).Iter() for iter.Scan(¬ifId) { e := cb.DeleteNotification(userId, notifId.String()) if e != nil { log.WithError(e).Warnf("[DeleteNotifications] failed to delete notif %s for user %s", notifId.String(), userId) } } if err := iter.Close(); err != nil { return err } return nil } func (cb *CassandraBackend) RetrieveNotification(userID, notifID string) (notif Notification, err error) { notifs, err := cb.SessionQuery(`SELECT * FROM notification WHERE user_id = ? AND notif_id = ?`, userID, notifID).Iter().SliceMap() if err != nil { return } if len(notifs) == 0 { err = errors.New("not found") return } notif = Notification{} notif.UnmarshalCQLMap(notifs[0]) return notif, nil } func (cb *CassandraBackend) DeleteNotification(userID, notifID string) error { return cb.SessionQuery(`DELETE FROM notification WHERE user_id = ? AND notif_id = ?`, userID, notifID).Exec() } ================================================ FILE: src/backend/main/go.backends/store/cassandra/participant_lookup.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package store import ( "errors" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/gocassa/gocassa" "strings" ) func (cb *CassandraBackend) LookupHash(user_id UUID, uri string) (hashes []HashLookup, err error) { rawHashes := []map[string]interface{}{} clean_uri := strings.ToLower(uri) rawHashes, err = cb.SessionQuery(`SELECT * FROM hash_lookup WHERE user_id = ? AND uri = ?`, user_id, clean_uri).Iter().SliceMap() if err != nil { return } if len(rawHashes) == 0 { err = errors.New("not found") return } for _, hash := range rawHashes { h := new(HashLookup) h.UnmarshalCQLMap(hash) hashes = append(hashes, *h) } return } func (cb *CassandraBackend) CreateHashLookup(participant HashLookup) error { lookupT := cb.IKeyspace.Table("hash_lookup", &HashLookup{}, gocassa.Keys{ PartitionKeys: []string{"user_id", "uri", "hash"}, }).WithOptions(gocassa.Options{TableName: "hash_lookup"}) // need to overwrite default gocassa table naming convention // save lookup err := lookupT.Set(participant).Run() if err != nil { return fmt.Errorf("[CassandraBackend] CreateHashLookup: %s", err) } return nil } ================================================ FILE: src/backend/main/go.backends/store/cassandra/providers.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package store import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "github.com/gocassa/gocassa" "gopkg.in/oleiade/reflections.v1" "time" ) func (cb *CassandraBackend) CreateProvider(provider *Provider) CaliopenError { if provider.DateInsert.IsZero() { provider.DateInsert = time.Now() } table := cb.IKeyspace.Table("provider", &Provider{}, gocassa.Keys{ PartitionKeys: []string{"name", "instance"}, }).WithOptions(gocassa.Options{TableName: "provider"}) err := table.Set(provider).Run() if err != nil { log.WithError(err).Errorf("[CassandraBackend] set fails for %+v", provider) return WrapCaliopenErrf(err, DbCaliopenErr, "[CassandraBackend] CreateProvider fails : %s", err.Error()) } return nil } func (cb *CassandraBackend) RetrieveProvider(name, instance string) (provider *Provider, err CaliopenError) { provider = new(Provider) m := map[string]interface{}{} q := cb.SessionQuery(`SELECT * FROM provider WHERE name = ? AND instance = ?`, name, instance) e := q.MapScan(m) if e != nil { return nil, WrapCaliopenErr(e, DbCaliopenErr, "") } provider.UnmarshalCQLmap(m) return } func (cb *CassandraBackend) UpdateProvider(provider *Provider, fields map[string]interface{}) (err CaliopenError) { // check if provider exists before executing UPDATE because `IF EXISTST` statement not supported by scylladb as of june 2019 if cb.SessionQuery(`SELECT name FROM providers WHERE name = ? AND instance = ?`, provider.Name, provider.Instance).Iter().NumRows() == 0 { return NewCaliopenErr(NotFoundCaliopenErr, "not found") } if len(fields) > 0 { //get cassandra's field name for each field to modify cassaFields := map[string]interface{}{} for field, value := range fields { cassaField, err := reflections.GetFieldTag(provider, field, "cql") if err != nil { return NewCaliopenErrf(UnprocessableCaliopenErr, "[CassandraBackend] UpdateProvider failed to find a cql field for object field %s", field) } if cassaField != "-" { cassaFields[cassaField] = value } } table := cb.IKeyspace.Table("provider", &Provider{}, gocassa.Keys{ PartitionKeys: []string{"name", "instance"}, }).WithOptions(gocassa.Options{TableName: "provider"}) e := table.Where(gocassa.Eq("name", provider.Name), gocassa.Eq("instance", provider.Instance)).Update(cassaFields).Run() if e != nil { log.WithError(e).Errorf("[CassandraBackend] UpdateProvider failed to UPDATE with fields %+v", cassaFields) return WrapCaliopenErr(e, DbCaliopenErr, "") } } return nil } func (cb *CassandraBackend) DeleteProvider(provider *Provider) CaliopenError { err := cb.SessionQuery(`DELETE FROM provider WHERE name = ? AND instance = ?`, provider.Name, provider.Instance).Exec() if err != nil { log.WithError(err).Errorf("[CassandraBackend] DeleteProvider failed for %+v", provider) return WrapCaliopenErr(err, DbCaliopenErr, "") } return nil } ================================================ FILE: src/backend/main/go.backends/store/cassandra/raw_messages.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package store import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "github.com/gocassa/gocassa" "github.com/gocql/gocql" "io" ) func (cb *CassandraBackend) StoreRawMessage(msg RawMessage) (err error) { rawMsgTable := cb.IKeyspace.MapTable("raw_message", "raw_msg_id", &RawMessage{}) consistency := gocql.Consistency(cb.CassandraConfig.Consistency) // need to overwrite default gocassa naming convention that add `_map_name` to the mapTable name rawMsgTable = rawMsgTable.WithOptions(gocassa.Options{ TableName: "raw_message", Consistency: &consistency, }) // handle emails too large to fit into cassandra if msg.Raw_Size > cb.CassandraConfig.SizeLimit { if cb.CassandraConfig.WithObjStore { uri, err := cb.ObjectsStore.PutRawMessage(msg.Raw_msg_id, msg.Raw_data) if err != nil { return err } msg.URI = uri msg.Raw_data = "" } else { return errors.New("Object too large to fit into cassandra") } } if err = rawMsgTable.Set(msg).Run(); err != nil { return err } return } // returns a RawMessage object, with 'raw_data' property always filled // (even if raw_data was stored outside of cassandra) func (cb *CassandraBackend) GetRawMessage(raw_message_id string) (message RawMessage, err error) { m := map[string]interface{}{} q := cb.SessionQuery(`SELECT * FROM raw_message WHERE raw_msg_id = ?`, raw_message_id) err = q.MapScan(m) if err != nil { return RawMessage{}, err } message.UnmarshalCQLMap(m) // check if raw_data is filled or if we need to get it from object store if message.URI != "" && len(message.Raw_data) == 0 { reader, e := cb.ObjectsStore.GetObject(message.URI) if e != nil { return RawMessage{}, e } raw_data := make([]byte, message.Raw_Size) s, e := reader.Read(raw_data) if s == 0 || e != io.EOF { return RawMessage{}, e } if uint64(s) != message.Raw_Size { log.Warnf("[cassandra.GetRawMessage] : Read %d bytes from Object Store, expected %d.", s, message.Raw_Size) } message.Raw_data = string(raw_data) } return } func (cb *CassandraBackend) SetDeliveredStatus(raw_msg_id string, delivered bool) error { return cb.SessionQuery(`UPDATE raw_message SET delivered = ? WHERE raw_msg_id = ?`, delivered, raw_msg_id).Exec() } ================================================ FILE: src/backend/main/go.backends/store/cassandra/related.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package store import ( "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/gocassa/gocassa" "github.com/mitchellh/hashstructure" "gopkg.in/oleiade/reflections.v1" "reflect" ) type relatedReference struct { Keys map[string]string Object HasTable Relations []gocassa.Relation Table string } // UpdateRelated ensure that tables related (joined) to obj are up to date in db, // ie related objects are updated, created or deleted accordingly. func (cb *CassandraBackend) UpdateRelated(new, old HasRelated, isNew bool) error { oldRelateds := map[uint64]relatedReference{} if isNew { MarshalRelated(new) } else { // build a map with old's relateds list to find if some relateds have been deleted in new for oldRelated := range old.GetSetRelated() { ref := relatedReference{ Keys: map[string]string{}, Object: oldRelated.(HasTable), Relations: []gocassa.Relation{}, } ref.Table, ref.Keys, _ = oldRelated.(HasTable).GetTableInfos() // build relations to select the right row relations := map[string]interface{}{} for property, key := range ref.Keys { value, err := reflections.GetField(oldRelated, property) if err == nil { ref.Relations = append(ref.Relations, gocassa.Eq(key, value)) relations[key] = value } } hash, err := hashstructure.Hash(relations, nil) if err == nil { oldRelateds[hash] = ref } } } for newRelated := range new.GetSetRelated() { if rel, ok := newRelated.(HasTable); ok { table, mapKeys, _ := rel.GetTableInfos() // build gocassa Table object keys := []string{} for _, key := range mapKeys { keys = append(keys, key) } T := cb.IKeyspace.Table(table, rel, gocassa.Keys{ PartitionKeys: keys, }).WithOptions(gocassa.Options{TableName: table}) // need to overwrite default gocassa table naming convention err := T.Set(rel).Run() if err != nil { return fmt.Errorf("[CassandraBackend] UpdateRelated: %s", err) } if !isNew { // remove keys from oldRelated // thus oldRelateds map will only holds remaining entries that are not in the new state relations := map[string]interface{}{} for property, key := range mapKeys { value, err := reflections.GetField(newRelated, property) if err == nil { relations[key] = value } } hash, err := hashstructure.Hash(relations, nil) if err == nil { delete(oldRelateds, hash) } } } } if len(oldRelateds) > 0 { // it remains relateds in the map, meaning these relateds have been removed from contact // need to delete related in joined table for _, related := range oldRelateds { // build gocassa Table object keys := []string{} for _, key := range related.Keys { keys = append(keys, key) } T := cb.IKeyspace.Table(related.Table, related.Object, gocassa.Keys{ PartitionKeys: keys, }).WithOptions(gocassa.Options{TableName: related.Table}) // need to overwrite default gocassa table naming convention // delete row err := T.Where(related.Relations...).Delete().Run() if err != nil { return fmt.Errorf("[CassandraBackend] UpdateRelated: %s", err) } } } return nil } // RetrieveRelated fetches rows from related (joined) table(s) and embeds rows into obj. func (cb *CassandraBackend) RetrieveRelated(obj HasRelated) error { for field, related := range obj.GetRelatedList() { if rel, ok := related.(HasTable); ok { table, partitionKeys, collectionKeys := rel.GetTableInfos() // build gocassa Table object keys := []string{} for _, key := range partitionKeys { keys = append(keys, key) } T := cb.IKeyspace.Table(table, rel, gocassa.Keys{ PartitionKeys: keys, }).WithOptions(gocassa.Options{TableName: table}) // need to overwrite default gocassa table naming convention // build relations to select right rows relations := []gocassa.Relation{} for property, key := range collectionKeys { value, err := reflections.GetField(obj, property) if err == nil && value != nil { relations = append(relations, gocassa.Eq(key, value)) } } // retrieve rows rows := new([]map[string]interface{}) err := T.Where(relations...).Read(rows).Run() if err != nil { return fmt.Errorf("[CassandraBackend] RetrieveRelated: %s", err) } // put rows (of unknown type at compilation) into obj embeddedSlice, err := reflections.GetField(obj, field) // Create a slice to begin with embeddedSliceType := reflect.TypeOf(embeddedSlice) toEmbed := reflect.MakeSlice(embeddedSliceType, 0, 1) // set capacity as needed // Create a pointer to the slice value p := reflect.New(toEmbed.Type()) // Set the pointer to the slice (otherwise, my_slice would be 'un-addressable') p.Elem().Set(toEmbed) if err == nil { for _, row := range *rows { rel.UnmarshalCQLMap(row) toEmbed = reflect.Append(toEmbed, reflect.Indirect(reflect.ValueOf(rel))) } } reflections.SetField(obj, field, toEmbed.Interface()) } } return nil } // DeleteRelated ensure that tables related (joined) to obj are up to date in db, // ie related objects are deleted accordingly. func (cb *CassandraBackend) DeleteRelated(obj HasRelated) error { for related := range obj.GetSetRelated() { if rel, ok := related.(HasTable); ok { table, mapKeys, _ := rel.GetTableInfos() // build gocassa Table object keys := []string{} for _, key := range mapKeys { keys = append(keys, key) } T := cb.IKeyspace.Table(table, rel, gocassa.Keys{ PartitionKeys: keys, }).WithOptions(gocassa.Options{TableName: table}) // need to overwrite default gocassa table naming convention // build relations to select the right row relations := []gocassa.Relation{} for property, key := range mapKeys { value, err := reflections.GetField(obj, property) if err == nil { relations = append(relations, gocassa.Eq(key, value)) } } // delete row err := T.Where(relations...).Delete().Run() if err != nil { return fmt.Errorf("[CassandraBackend] UpdateRelated: %s", err) } } } return nil } ================================================ FILE: src/backend/main/go.backends/store/cassandra/settings.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package store import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) func (cb *CassandraBackend) GetSettings(user_id string) (settings *Settings, err error) { settings = new(Settings).NewEmpty().(*Settings) m := map[string]interface{}{} q := cb.SessionQuery(`SELECT * FROM settings WHERE user_id = ?`, user_id) err = q.MapScan(m) if err != nil { return nil, err } settings.UnmarshalCQLMap(m) return settings, err } ================================================ FILE: src/backend/main/go.backends/store/cassandra/tags.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package store import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/gocql/gocql" "time" ) // retrieve tags of type 'user' belonging to user_id func (cb *CassandraBackend) RetrieveUserTags(user_id string) (tags []Tag, err error) { all_tags, err := cb.SessionQuery(`SELECT * FROM user_tag WHERE user_id = ?`, user_id).Iter().SliceMap() if err != nil { return } if len(all_tags) == 0 { err = errors.New("tags not found") return } for _, tag := range all_tags { t := new(Tag) t.UnmarshalCQLMap(tag) tags = append(tags, *t) } return } // CreateTag inserts Tag into db func (cb *CassandraBackend) CreateTag(tag *Tag) error { user_id, _ := gocql.UUIDFromBytes((*tag).User_id.Bytes()) (*tag).Date_insert = time.Now() (*tag).Type = TagType(UserTag) return cb.SessionQuery(`INSERT INTO user_tag (user_id, name, date_insert, importance_level, label, type) VALUES (?,?,?,?,?,?)`, user_id, (*tag).Name, (*tag).Date_insert, (*tag).Importance_level, (*tag).Label, (*tag).Type).Exec() } func (cb *CassandraBackend) RetrieveTag(user_id, name string) (tag Tag, err error) { tags, err := cb.SessionQuery(`SELECT * FROM user_tag WHERE user_id = ? AND name = ?`, user_id, name).Iter().SliceMap() if err != nil { return } if len(tags) == 0 { err = errors.New("tag not found") return } tag = Tag{} err = tag.UnmarshalCQLMap(tags[0]) if err != nil { return } return tag, nil } func (cb *CassandraBackend) UpdateTag(tag *Tag) error { return cb.SessionQuery(`UPDATE user_tag SET importance_level = ?, label = ?, type = ? WHERE user_id = ? AND name = ?`, tag.Importance_level, tag.Label, tag.Type, tag.User_id, tag.Name, ).Exec() } func (cb *CassandraBackend) DeleteTag(user_id, name string) error { return cb.SessionQuery(`DELETE FROM user_tag WHERE user_id = ? AND name = ?`, user_id, name).Exec() } ================================================ FILE: src/backend/main/go.backends/store/cassandra/usernames.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. // package store import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/helpers" log "github.com/Sirupsen/logrus" "github.com/gocql/gocql" "strings" ) // UserNameStorage interface implementation for cassandra func (cb *CassandraBackend) UsernameIsAvailable(username string) (resp bool, err error) { resp = false err = nil lookup := helpers.EscapeUsername(username) found := map[string]interface{}{} err = cb.SessionQuery(`SELECT COUNT(*) FROM user_name WHERE name = ?`, strings.ToLower(lookup)).MapScan(found) if err != nil { log.WithError(err).Infof("username lookup error : %v", err) return } if found["count"].(int64) != 0 { return } resp = true return } // UserByUsername lookups table user_name to get the user_id for the given username // if a user_id is found, the user is fetched from user table. func (cb *CassandraBackend) UserByUsername(username string) (user *User, err error) { user_id := new(gocql.UUID) err = cb.SessionQuery(`SELECT user_id FROM user_name WHERE name = ?`, strings.ToLower(username)).Scan(user_id) if err != nil || len(user_id.Bytes()) == 0 { return nil, err } return cb.RetrieveUser(user_id.String()) } ================================================ FILE: src/backend/main/go.backends/store/cassandra/users.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. // // UserStorage interface implementation for cassandra backend package store import ( "errors" "fmt" "time" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/gocassa/gocassa" "github.com/gocql/gocql" "gopkg.in/oleiade/reflections.v1" ) func (cb *CassandraBackend) RetrieveUser(user_id string) (user *User, err error) { u, err := cb.SessionQuery(`SELECT * FROM user WHERE user_id = ?`, user_id).Iter().SliceMap() if err != nil { return nil, err } if len(u) != 1 { err = errors.New("[CassandraBackend] user not found") return nil, err } user = new(User) user.UnmarshalCQLMap(u[0]) return user, nil } func (cb *CassandraBackend) UpdateUser(user *User, fields map[string]interface{}) error { //get cassandra's field name for each field to modify cassaFields := map[string]interface{}{} for field, value := range fields { cassaField, err := reflections.GetFieldTag(user, field, "cql") if err != nil { return fmt.Errorf("[CassandraBackend] UpdateMessage failed to find a cql field for object field %s", field) } cassaFields[cassaField] = value } userT := cb.IKeyspace.Table("user", &User{}, gocassa.Keys{ PartitionKeys: []string{"user_id"}, }).WithOptions(gocassa.Options{TableName: "user"}) return userT.Where(gocassa.Eq("user_id", user.UserId.String())).Update(cassaFields).Run() } func (cb *CassandraBackend) UpdateUserPasswordHash(user *User) error { return cb.SessionQuery(`UPDATE user SET password = ?, privacy_features = ? WHERE user_id = ?`, user.Password, user.PrivacyFeatures, user.UserId, ).Exec() } // UserByRecoveryEmail lookups table user_recovery_email to get the user_id for the given email // if a user_id is found, the user is fetched from user table. func (cb *CassandraBackend) UserByRecoveryEmail(email string) (user *User, err error) { user_id := new(gocql.UUID) err = cb.SessionQuery(`SELECT user_id FROM user_recovery_email WHERE recovery_email = ?`, email).Scan(user_id) if err != nil || len(user_id.Bytes()) == 0 { return nil, err } return cb.RetrieveUser(user_id.String()) } // DeleteUser sets the date_delete in the database func (cb *CassandraBackend) DeleteUser(user_id string) error { return cb.SessionQuery(`UPDATE user SET date_delete = ? WHERE user_id = ?`, time.Now(), user_id).Exec() } // GetShardForUser returns user's shard_id or empty string if error func (cb *CassandraBackend) GetShardForUser(userID string) string { var shardID string err := cb.SessionQuery(`SELECT shard_id FROM user WHERE user_id = ?`, userID).Scan(&shardID) if err != nil { return "" } return shardID } ================================================ FILE: src/backend/main/go.backends/store/object_store/attachments.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package object_store import ( "io" ) func (mb *MinioBackend) PutAttachment(attchId string, attch io.Reader) (uri string, size int64, err error) { return mb.PutObject(attchId, mb.AttachmentBucket, attch) } ================================================ FILE: src/backend/main/go.backends/store/object_store/minio.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package object_store import ( obj "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/Sirupsen/logrus" "github.com/minio/minio-go" "io" ) type ( MinioBackend struct { OSSConfig Client *minio.Client } OSSConfig struct { Endpoint string AccessKey string SecretKey string Location string RawMsgBucket string AttachmentBucket string } ObjectsStore interface { PutRawMessage(message_uuid obj.UUID, raw_message string) (uri string, err error) PutAttachment(attchId string, attch io.Reader) (uri string, size int64, err error) RemoveObject(uri string) error GetObject(uri string) (file io.Reader, err error) StatObject(uri string) (info minio.ObjectInfo, err error) } ) func InitializeObjectsStore(config OSSConfig) (oss ObjectsStore, err error) { mb := new(MinioBackend) mb.OSSConfig = config // Initialize minio client object. mb.Client, err = minio.NewWithRegion(config.Endpoint, config.AccessKey, config.SecretKey, false, config.Location) // or NewWithCredentials to avoid putting credentials directly into conf. file ? if err != nil { mb.Client = nil return } // Check to see if we already own RawMsgBucket (which happens if bucket already created by this client) exists, err := mb.Client.BucketExists(config.RawMsgBucket) if err != nil || !exists { // Create a new bucket for raw messages err = mb.Client.MakeBucket(config.RawMsgBucket, config.Location) if err != nil { logrus.WithError(err).Warnf("[ObjectStore] failed to create new bucket for large raw messages") mb.Client = nil return } } // Check to see if we already own AttachmentBucket (which happens if bucket already created by this client) exists, err = mb.Client.BucketExists(config.AttachmentBucket) if err != nil || !exists { // Create a new bucket for raw messages err = mb.Client.MakeBucket(config.AttachmentBucket, config.Location) if err != nil { logrus.WithError(err).Warnf("[ObjectStore] failed to create new bucket for draft attachments") mb.Client = nil return } } return mb, err } ================================================ FILE: src/backend/main/go.backends/store/object_store/objects.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package object_store import ( "fmt" "github.com/minio/minio-go" "io" "net/url" ) func (mb *MinioBackend) PutObject(name, bucket string, object io.Reader) (uri string, size int64, err error) { const uriTemplate = "s3://%s/%s" size, err = mb.Client.PutObject(bucket, name, object, -1, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { return "", 0, err } return fmt.Sprintf(uriTemplate, bucket, name), size, nil } func (mb *MinioBackend) RemoveObject(objURI string) error { uri, err := url.Parse(objURI) if err != nil || len(uri.Host) < 1 || len(uri.Path) < 2 { return err } return mb.Client.RemoveObject(uri.Host, uri.Path[1:]) } func (mb *MinioBackend) GetObject(objURI string) (file io.Reader, err error) { uri, err := url.Parse(objURI) if err != nil || len(uri.Host) < 1 || len(uri.Path) < 2 { return nil, err } return mb.Client.GetObject(uri.Host, uri.Path[1:], minio.GetObjectOptions{}) } func (mb *MinioBackend) StatObject(objURI string) (info minio.ObjectInfo, err error) { uri, err := url.Parse(objURI) if err != nil || len(uri.Host) < 1 || len(uri.Path) < 2 { return } return mb.Client.StatObject(uri.Host, uri.Path[1:], minio.StatObjectOptions{}) } ================================================ FILE: src/backend/main/go.backends/store/object_store/raw_messages.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package object_store import ( obj "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "strings" ) func (mb *MinioBackend) PutRawMessage(message_uuid obj.UUID, raw_email string) (uri string, err error) { email_reader := strings.NewReader(raw_email) uri, _, err = mb.PutObject(message_uuid.String(), mb.RawMsgBucket, email_reader) return } ================================================ FILE: src/backend/main/go.backends/store/vault/credentials.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package vault import ( "errors" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type VaultCredentials interface { CreateCredentials(userIdentity *UserIdentity, cred Credentials) error RetrieveCredentials(userId, remoteId string) (Credentials, error) UpdateCredentials(userId, remoteId string, cred Credentials, upsertMode bool) error DeleteCredentials(userId, remoteId string) error } func (vault *HVaultClient) CreateCredentials(userIdentity *UserIdentity, cred Credentials) error { return vault.UpdateCredentials(userIdentity.UserId.String(), userIdentity.Id.String(), cred, true) } // RetrieveCrendentials gets credentials from vault, if application has read rights on vault func (vault *HVaultClient) RetrieveCredentials(userId, remoteId string) (cred Credentials, err error) { cred = Credentials{} path := fmt.Sprintf(credentialsPath, userId, remoteId) secret, err := vault.hclient.Logical().Read(path) if err != nil { return } if secret == nil || secret.Data == nil { // secret not found err = errors.New("secret not found") return } data, ok := secret.Data["data"] if !ok { err = errors.New("secret not found") return } // Credentials is a map[string]string // only copy values from secret.Data that are strings for k, v := range data.(map[string]interface{}) { switch v.(type) { case string: cred[k] = v.(string) default: continue } } return } func (vault *HVaultClient) UpdateCredentials(userId, remoteId string, cred Credentials, upsertMode bool) error { payload := make(map[string]interface{}) payload["data"] = cred path := fmt.Sprintf(credentialsPath, userId, remoteId) if !upsertMode { // ensure credentials exist before updating // or silently returns without doing anything (mimicking cassandra) secret, err := vault.hclient.Logical().Read(path) if err != nil || secret == nil { return nil } } _, err := vault.hclient.Logical().Write(path, payload) return err } func (vault *HVaultClient) DeleteCredentials(userId, remoteId string) error { path := fmt.Sprintf(credentialsPath, userId, remoteId) _, err := vault.hclient.Logical().Delete(path) return err } ================================================ FILE: src/backend/main/go.backends/store/vault/hvault_interface.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ // interfaces definition to Hashicorp Vault server package vault // As of june 2018, only one interface for CRUD operation on credentials. Later on, we may add Cubbyhole secrets engine, databases secret engine and so on… type HVault interface { VaultCredentials } ================================================ FILE: src/backend/main/go.backends/store/vault/vault_client.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ // vault_client provides an interface to interact with HashiCorp Vault server package vault import ( "fmt" hvault "github.com/hashicorp/vault/api" ) type HVaultClient struct { hclient *hvault.Client HVaultConfig } type HVaultConfig struct { Url string `mapstructure:"url"` Username string `mapstructure:"username"` Password string `mapstructure:"password"` } const credentialsPath = "secret/data/remoteid/credentials/%s/%s" // path to store credentials => secret/data/remoteid/credentials/user_id/remote_id const loginPath = "auth/userpass/login/%s" // InitializeVaultBackend checks if a Vault server is available and returns an authenticated VaultClient func InitializeVaultBackend(hvConf HVaultConfig) (hv HVault, err error) { config := hvault.DefaultConfig() config.Address = hvConf.Url var hc HVaultClient hc.Url = hvConf.Url hc.Username = hvConf.Username hc.Password = hvConf.Password hc.hclient, err = hvault.NewClient(config) if err != nil { return nil, err } //authentication with user/password method options := map[string]interface{}{ "password": hvConf.Password, } path := fmt.Sprintf(loginPath, hvConf.Username) secret, err := hc.hclient.Logical().Write(path, options) if err != nil { return nil, err } hc.hclient.SetToken(secret.Auth.ClientToken) //TODO: manage token expiration (default TTL is 32 days) _, err = hc.hclient.Sys().Health() if err != nil { return nil, err } return &hc, nil } ================================================ FILE: src/backend/main/go.main/caliopen.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package caliopen import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/Messaging" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/Notifications" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/REST" log "github.com/Sirupsen/logrus" "github.com/nats-io/go-nats" ) var ( Facilities *CaliopenFacilities ) type ( CaliopenFacilities struct { config CaliopenConfig RESTfacility REST.RESTservices Cache backends.APICache // NATS facility nats *nats.Conn MessagingFacility Messaging.Facility // LDA facility LDAstore backends.LDAStore // Notifications facility Notifiers Notifications.Notifiers // Admin user on whose behalf actions could be done Admin *User } ) func Initialize(config CaliopenConfig) error { Facilities = new(CaliopenFacilities) return Facilities.initialize(config) } func (facilities *CaliopenFacilities) initialize(config CaliopenConfig) (err error) { facilities.config = config // NATS facility initialization facilities.nats, err = nats.Connect(config.NatsConfig.Url) if err != nil { log.WithError(err).Error("CaliopenFacilities : initalization of NATS connexion failed") return } // REST facility initialization rest := REST.NewRESTfacility(config, facilities.nats) facilities.RESTfacility = rest // copy cache facility from REST facility facilities.Cache = rest.Cache // Notifications facility initialization notifier := Notifications.NewNotificationsFacility(config, facilities.nats) facilities.Notifiers = notifier // Messaging facility initialization facilities.MessagingFacility, err = Messaging.NewCaliopenMessaging(config, notifier) if err != nil { log.WithError(err).Error("CaliopenFacilities : initalization of Messaging facility failed") return } return } ================================================ FILE: src/backend/main/go.main/contact/contact.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package contact import ( "bytes" "encoding/base64" "errors" "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/helpers" log "github.com/Sirupsen/logrus" "github.com/emersion/go-vcard" "github.com/keybase/go-crypto/openpgp" "github.com/satori/go.uuid" "strings" ) // Parse EMAIL vcard field func parseEmail(field *vcard.Field) *objects.EmailContact { email := new(objects.EmailContact) uid := new(objects.UUID) _ = uid.UnmarshalBinary(uuid.NewV4().Bytes()) email.EmailId = *uid email.Address = strings.ToLower(field.Value) if field.Params["TYPE"] != nil { email.Type = strings.ToLower(field.Params["TYPE"][0]) } // TODO complete struct filling email.Label = field.Value return email } // Parse TEL vcard field func parsePhone(field *vcard.Field) *objects.Phone { phone := new(objects.Phone) uid := new(objects.UUID) _ = uid.UnmarshalBinary(uuid.NewV4().Bytes()) phone.PhoneId = *uid phone.Number = field.Value if field.Params["TYPE"] != nil { phone.Type = strings.ToLower(field.Params["TYPE"][0]) } return phone } // Parse IMPP vcard field func parseIm(field *vcard.Field) *objects.IM { im := new(objects.IM) uid := new(objects.UUID) _ = uid.UnmarshalBinary(uuid.NewV4().Bytes()) im.IMId = *uid im.Address = strings.ToLower(field.Value) im.Label = field.Value if field.Params["TYPE"] != nil { im.Type = strings.ToLower(field.Params["TYPE"][0]) } return im } // Read an armored PGP public key and return an openpgp.Entity structure func readPgpKey(pubkey []byte) (*openpgp.Entity, error) { reader := bytes.NewReader(pubkey) var entitiesList openpgp.EntityList var err error entitiesList, err = openpgp.ReadArmoredKeyRing(reader) if err != nil { return nil, err } //handle only first key found for now if len(entitiesList) > 1 { return nil, errors.New("More than one key found in payload") } return entitiesList[0], nil } // Parse KEY vcard field and transform to a PublicKey that belong to a contact func parseKey(field *vcard.Field, contact *objects.Contact) (*objects.PublicKey, error) { key := new(objects.PublicKey) var err error if field.Params["TYPE"] != nil && field.Params["TYPE"][0] == "PGP" { key.KeyType = "pgp" } if field.Params["ENCODING"] != nil && field.Params["ENCODING"][0] == "b" { pubkey, err := base64.StdEncoding.DecodeString(field.Value) if err != nil { return &objects.PublicKey{}, err } entity, err := readPgpKey(pubkey) if err != nil { return &objects.PublicKey{}, err } err = key.UnmarshalPGPEntity("PGP key", entity, contact) } else { return &objects.PublicKey{}, errors.New("Unknow key encoding") } log.Info("Have parsed PGP key ", key.Fingerprint, " with algorithm ", key.Algorithm) return key, err } // Parse ADR vcard field func parseAddress(addr *vcard.Address) *objects.PostalAddress { address := new(objects.PostalAddress) uid := new(objects.UUID) _ = uid.UnmarshalBinary(uuid.NewV4().Bytes()) address.AddressId = *uid address.City = addr.Locality address.Country = addr.Country address.Region = addr.Region address.PostalCode = addr.PostalCode address.Street = addr.StreetAddress return address } // Transform a vcard into an objects.Contact structure func FromVcard(user *objects.UserInfo, card vcard.Card) (*objects.Contact, error) { contact := new(objects.Contact).NewEmpty().(*objects.Contact) contact.UserId = objects.UUID(uuid.FromStringOrNil(user.User_id)) contact.Title = card.PreferredValue(vcard.FieldFormattedName) if card.Name() != nil { contact.FamilyName = card.Name().FamilyName contact.GivenName = card.Name().GivenName } // check version // TODO implement version 4.0 (rfc 6350) version := card[vcard.FieldVersion] if version != nil { if !(version[0].Value == "3.0" || version[0].Value == "4.0") { return contact, errors.New("Only vcard format 3.0 and 4.0 are supported") } } // emails emails := card[vcard.FieldEmail] if emails != nil { contact.Emails = []objects.EmailContact{} for _, email := range emails { e := parseEmail(email) if e.Address != "" { contact.Emails = append(contact.Emails, *e) } } } // phones phones := card[vcard.FieldTelephone] if phones != nil { contact.Phones = []objects.Phone{} for _, phone := range phones { p := parsePhone(phone) if p.Number != "" { contact.Phones = append(contact.Phones, *p) } } } helpers.NormalizePhoneNumbers(contact) ims := card[vcard.FieldIMPP] if ims != nil { contact.Ims = []objects.IM{} for _, im := range ims { i := parseIm(im) if i.Address != "" { contact.Ims = append(contact.Ims, *i) } } } // addresses addrs := card.Addresses() if addrs != nil { contact.Addresses = []objects.PostalAddress{} for _, addr := range addrs { a := parseAddress(addr) contact.Addresses = append(contact.Addresses, *a) } } // public keys keys := card[vcard.FieldKey] if keys != nil { contact.PublicKeys = make([]objects.PublicKey, 0, len(keys)) for _, key := range keys { k, err := parseKey(key, contact) if err != nil { log.Warn("Error during vcard KEY parsing ", err) } else { contact.PublicKeys = append(contact.PublicKeys, *k) } } } /* TODO: Need to change index mappings of contact.infos infos := make(map[string]string) if uid := card[vcard.FieldUID]; uid != nil { infos["uid"] = uid[0].Value } if rev := card[vcard.FieldRevision]; rev != nil { infos["revision"] = rev[0].Value } contact.Infos = make(map[string]string) for k, v := range infos { contact.Infos[k] = v } */ return contact, nil } ================================================ FILE: src/backend/main/go.main/contact/contact_test.go ================================================ package contact import ( "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "strings" "testing" ) // https://en.wikipedia.org/wiki/VCard#vCard_2.1 var testWikipediav2_1 = `BEGIN:VCARD VERSION:2.1 N:Gump;Forrest;;Mr. FN:Forrest Gump ORG:Bubba Gump Shrimp Co. TITLE:Shrimp Man PHOTO;GIF:http://www.example.com/dir_photos/my_photo.gif TEL;WORK;VOICE:(111) 555-1212 TEL;HOME;VOICE:(404) 555-1212 ADR;WORK;PREF:;;100 Waters Edge;Baytown;LA;30314;United States of America LABEL;WORK;PREF;ENCODING=QUOTED-PRINTABLE;CHARSET=UTF-8:100 Waters Edge=0D= =0ABaytown\, LA 30314=0D=0AUnited States of America ADR;HOME:;;42 Plantation St.;Baytown;LA;30314;United States of America LABEL;HOME;ENCODING=QUOTED-PRINTABLE;CHARSET=UTF-8:42 Plantation St.=0D=0A= Baytown, LA 30314=0D=0AUnited States of America EMAIL:forrestgump@example.com REV:20080424T195243Z END:VCARD` // https://en.wikipedia.org/wiki/VCard#vCard_3.0 var testWikipediav3 = `BEGIN:VCARD VERSION:3.0 N:Gump;Forrest;;Mr.; FN:Forrest Gump ORG:Bubba Gump Shrimp Co. TITLE:Shrimp Man PHOTO;VALUE=URI;TYPE=GIF:;http://www.example.com/dir_photos/my_photo.gif TEL;TYPE=WORK,VOICE:(111) 555-1212 TEL;TYPE=HOME,VOICE:(404) 555-1212 ADR;TYPE=WORK,PREF:;;100 Waters Edge;Baytown;LA;30314;United States of America LABEL;TYPE=WORK,PREF:100 Waters Edge\nBaytown\, LA 30314\nUnited States of America ADR;TYPE=HOME:;;42 Plantation St.;Baytown;LA;30314;United States of America LABEL;TYPE=HOME:42 Plantation St.\nBaytown\, LA 30314\nUnited States of America EMAIL:forrestgump@example.com REV:2008-04-24T19:52:43Z END:VCARD` // https://en.wikipedia.org/wiki/VCard#vCard_4.0 var testWikipediav4 = `BEGIN:VCARD VERSION:4.0 N:Gump;Forrest;;Mr.; FN:Forrest Gump ORG:Bubba Gump Shrimp Co. TITLE:Shrimp Man PHOTO;MEDIATYPE=image/gif:http://www.example.com/dir_photos/my_photo.gif TEL;TYPE=work,voice;VALUE=uri:tel:+1-111-555-1212 TEL;TYPE=home,voice;VALUE=uri:tel:+1-404-555-1212 ADR;TYPE=WORK;PREF=1;LABEL="100 Waters Edge\nBaytown\, LA 30314\nUnited States of America":;;100 Waters Edge;Baytown;LA;30314;United St ates of America ADR;TYPE=HOME;LABEL="42 Plantation St.\nBaytown\, LA 30314\nUnited States of America":;;42 Plantation St.;Baytown;LA;30314;United State s of America EMAIL:forrestgump@example.com REV:20080424T195243Z x-qq:21588891 END:VCARD` // Format 2.1 used for v3 var testInvalidv3 = `begin:vcard fn:Emma email;internet:Emma@tomme.de.savoie version:3.0 end:vcard ` var validTests = []struct { s string }{ {testWikipediav3}, {testWikipediav4}, } var invalidFormat = []struct { s string }{ {testWikipediav2_1}, } func TestFromVcardValid(t *testing.T) { info := objects.UserInfo{User_id: "ede04443-b60f-4869-9040-20bd6b1e33c1"} for _, test := range validTests { r := strings.NewReader(test.s) cards, err := ParseVcardFile(r) if len(cards) != 1 { t.Error("Expected only one vcard") } contact, err := FromVcard(&info, cards[0]) if err != nil { t.Error("Expecting null error ", err) } if len(contact.Emails) == 0 { t.Error("No email found in test vcard ") } if len(contact.Phones) == 0 { t.Error("No phone found in test vcard ") } } } func TestFromVcardInvalid(t *testing.T) { info := objects.UserInfo{User_id: "ede04443-b60f-4869-9040-20bd6b1e33c1"} for _, test := range invalidFormat { r := strings.NewReader(test.s) cards, err := ParseVcardFile(r) if len(cards) != 1 { t.Error("Expected only one vcard") } _, err = FromVcard(&info, cards[0]) if err == nil { t.Error("Expecting not null error ") } } } // Test testInvalidv3 card func TestFromVcardInvalidEmailFormat(t *testing.T) { info := objects.UserInfo{User_id: "ede04443-b60f-4869-9040-20bd6b1e33c1"} r := strings.NewReader(testInvalidv3) cards, err := ParseVcardFile(r) if len(cards) != 1 { t.Error("Expected only one vcard") } contact, err := FromVcard(&info, cards[0]) if err != nil { t.Error("Expecting no error ") } if len(contact.Emails) > 0 { t.Error("Expecting no email in invalid vcard") } } ================================================ FILE: src/backend/main/go.main/contact/vcard.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package contact import ( "errors" "github.com/emersion/go-vcard" "io" ) // Parse .vcf .vcard file, returting list of Card objects func ParseVcardFile(file io.Reader) ([]vcard.Card, error) { cards := make([]vcard.Card, 0, 5) dec := vcard.NewDecoder(file) for { card, err := dec.Decode() if err == io.EOF { break } else if err != nil { return []vcard.Card{}, err } cards = append(cards, card) } if len(cards) == 0 { return cards, errors.New("No vcard found in file") } return cards, nil } ================================================ FILE: src/backend/main/go.main/facilities/Messaging/facility.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package Messaging import ( "encoding/json" "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/Notifications" log "github.com/Sirupsen/logrus" "github.com/nats-io/go-nats" "github.com/satori/go.uuid" ) type Facility interface { HandleUserAction(msg *nats.Msg) } type CaliopenMessaging struct { caliopenNotifier *Notifications.Notifier natsConn *nats.Conn Subscriptions map[string]*nats.Subscription } // unexported vars to help override funcs in tests var ( notifyByEmail = func(notifier *Notifications.Notifier, notif *Notification) CaliopenError { return notifier.ByEmail(notif) } ) func NewCaliopenMessaging(config CaliopenConfig, notifier *Notifications.Notifier) (Facility, error) { if notifier == nil { return nil, errors.New("[NewCaliopenMessaging] needs a non-nil notifier") } if notifier.NatsQueue == nil { return nil, errors.New("[NewCaliopenMessaging] needs a non-nil natsConn") } cm := new(CaliopenMessaging) cm.natsConn = notifier.NatsQueue cm.Subscriptions = map[string]*nats.Subscription{} if config.NatsConfig.Users_topic == "" { return nil, errors.New("[NewCaliopenMessaging] wont subscribe to empty topic") } userSub, err := cm.natsConn.QueueSubscribe(config.NatsConfig.Users_topic, config.NatsConfig.NatsQueue, cm.HandleUserAction) if err != nil { log.WithError(err).Errorf("[NewCaliopenMessaging] failed to subscribe to %s topic on NATS", config.NatsConfig.Users_topic) return nil, err } cm.Subscriptions[config.NatsConfig.Users_topic] = userSub cm.caliopenNotifier = notifier return cm, nil } func (cm *CaliopenMessaging) HandleUserAction(msg *nats.Msg) { payload := new(struct { Message string `json:"message"` UserName string `json:"user_name"` UserId string `json:"user_id"` }) err := json.Unmarshal(msg.Data, payload) if err != nil { log.WithError(err).Errorf("[HandleUserAction] failed to unmarshal this nats' payload : %+v", msg.Data) return } switch payload.Message { case "created": user, err := cm.caliopenNotifier.Store.UserByUsername(payload.UserName) if err != nil { log.WithError(err).Errorf("[HandleUserAction] failed to retrieve user %s", payload.UserName) return } notif := &Notification{ NotifId: UUID(uuid.NewV1()), Type: OnboardingMails, User: user, } cErr := notifyByEmail(cm.caliopenNotifier, notif) if cErr != nil { log.WithError(cErr).Warnf("[HandleUserAction] ByEmail notification failed (code : %d, cause : %s)", cErr.Code(), cErr.Cause()) } default: log.Errorf("[HandleUserAction] unhandled message : %s", payload.Message) } } ================================================ FILE: src/backend/main/go.main/facilities/Messaging/facility_test.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package Messaging import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/NATS/go.mockednats" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/backendstest" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/Notifications" "github.com/nats-io/go-nats" "testing" ) func TestNewCaliopenMessaging(t *testing.T) { // test errors returns facility, err := NewCaliopenMessaging(CaliopenConfig{}, nil) if facility != nil { t.Errorf("expected a nil facility, got %+v", facility) } if err == nil { t.Error("NewCaliopenMessaging should return an error, got nil") } facility, err = NewCaliopenMessaging(CaliopenConfig{}, &Notifications.Notifier{}) if facility != nil { t.Errorf("expected a nil facility, got %+v", facility) } if err == nil { t.Error("initializing caliopen messaging with empty notifier should return an error, got nil") } natsServer, natsConn, err := mockednats.GetNats() defer natsServer.Shutdown() facility, err = NewCaliopenMessaging(CaliopenConfig{}, &Notifications.Notifier{ NatsQueue: natsConn, }) if err == nil { t.Error("initializing caliopen messaging with empty configuration should return error, got nil") } // test correct initialization notifier := &Notifications.Notifier{NatsQueue: natsConn} facility, err = NewCaliopenMessaging(CaliopenConfig{NatsConfig: NatsConfig{Users_topic: "userAction"}}, notifier) if err != nil { t.Error(err) } if f, ok := facility.(*CaliopenMessaging); ok { if len(f.Subscriptions) != 1 { t.Errorf("expected a facility with one nats subscription, got %d", len(f.Subscriptions)) } else if s, ok := f.Subscriptions["userAction"]; ok { if !s.IsValid() { t.Error("expected a valid subscription at Subscriptions[\"UserAction\"], got invalid") } } else { t.Error("expected to have a entry in CaliopenMessaging's Subscriptions at key `userAction`, got nothing") } if f.caliopenNotifier != notifier { t.Errorf("expected to have a CaliopenMessaging with the notifier passed in constructor (%p), got %p", &f.caliopenNotifier, ¬ifier) } } else { t.Errorf("expected NewCaliopenMessagin to return a *CaliopenMessaging struct, got %T", f) } // last error checking natsConn.Close() facility, err = NewCaliopenMessaging(CaliopenConfig{}, &Notifications.Notifier{ NatsQueue: natsConn, // passing closed connexion on purpose }) if err == nil { t.Error("initializing caliopen messaging with empty nats connexion should return an error, got nil") } } func TestCaliopenMessaging_HandleUserAction(t *testing.T) { natsServer, natsConn, err := mockednats.GetNats() defer natsServer.Shutdown() store, _ := backendstest.GetNotificationsBackends() notifier := &Notifications.Notifier{ NatsQueue: natsConn, Store: store, } facility, err := NewCaliopenMessaging(CaliopenConfig{NatsConfig: NatsConfig{Users_topic: "userAction"}}, notifier) if err != nil { t.Error(err) } // overring notifyByEmail func // because this test checks if the func is correctly called within HandleUserAction, nothing more var notifCalled bool notifyByEmail = func(notifier *Notifications.Notifier, notif *Notification) CaliopenError { notifCalled = true return nil } // test errors handling // invalid nats msg facility.HandleUserAction(&nats.Msg{}) if notifCalled { t.Error("expected that calling HandleUserAction with an empty nats message will not trigger notifyByEmail, but func was called") notifCalled = false } // unknown username facility.HandleUserAction(&nats.Msg{ Subject: "test", Reply: "test_reply", Data: []byte(`{"message":"created", "user_name": "unknown", "user_id": "7f8329c4-e220-45fc-89b2-d8535df69e83"}`), // invalid user }) if notifCalled { t.Error("expected that calling HandleUserAction with unknown username will not trigger notifyByEmail, but func was called") notifCalled = false } // unknown message facility.HandleUserAction(&nats.Msg{ Subject: "test", Reply: "test_reply", Data: []byte(`{"message":"unknown", "user_name": "emma", "user_id": "7f8329c4-e220-45fc-89b2-d8535df69e83"}`), // invalid user }) if notifCalled { t.Error("expected that calling HandleUserAction with unknown username will not trigger notifyByEmail, but func was called") notifCalled = false } // test valid payload facility.HandleUserAction(&nats.Msg{ Subject: "test", Reply: "test_reply", Data: []byte(`{"message":"created", "user_name": "emma", "user_id": "7f8329c4-e220-45fc-89b2-d8535df69e83"}`), }) if !notifCalled { t.Error("expected HandleUserAction to call notifyByEmail, but func was not called") } } ================================================ FILE: src/backend/main/go.main/facilities/Notifications/batch.go ================================================ package Notifications import ( "encoding/json" "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/satori/go.uuid" "strconv" "strings" "sync" ) type BatchNotification struct { emitter string locker *sync.Mutex notifications []Notification notificationsCount int notificationsThreshold int } const notificationsThreshold = 20 type BatchNotifier interface { Add(Notification) Save(notifier Notifiers, reference, ttl string) } func NewBatch(emitter string) *BatchNotification { return &BatchNotification{ emitter: emitter, locker: new(sync.Mutex), notificationsThreshold: notificationsThreshold, } } func (bn *BatchNotification) Add(n Notification) { bn.locker.Lock() bn.notificationsCount++ if bn.notificationsCount <= bn.notificationsThreshold { bn.notifications = append(bn.notifications, n) } bn.locker.Unlock() } // Save aggregates notifications into a single one with sub-notifications into its Body as a json array // then saves this notification in user's cassandra queue func (bn *BatchNotification) Save(notifier Notifiers, reference, ttl string) { notif, err := bn.aggregate(reference, ttl) if err == nil { notifier.ByNotifQueue(¬if) } } // aggregate flatten notifications into a single Notification // children are embedded in Notification.Body as a json array if they are less than batchThreshold // otherwise, only children_count is written func (bn *BatchNotification) aggregate(reference, ttl string) (Notification, error) { if len(bn.notifications) == 0 { return Notification{}, errors.New("[BatchNotifier] elements is empty") } notif := Notification{ Emitter: bn.emitter, NotifId: UUID(uuid.NewV1()), Reference: reference, TTLcode: ttl, Type: bn.notifications[0].Type, User: bn.notifications[0].User, ChildrenCount: bn.notificationsCount, } children := make([]NotificationModel, 0, len(bn.notifications)) if bn.notificationsCount <= bn.notificationsThreshold { for _, n := range bn.notifications { if notif.User == nil || n.User == nil || (notif.User.UserId.String() != n.User.UserId.String()) { return Notification{}, errors.New("[BatchNotifier] can't aggregate notifications : inconsistent user ids within notifications slice") } children = append(children, NotificationModel{ Body: n.Body, }) } } body := strings.Builder{} body.WriteString(`{"size":`) body.WriteString(strconv.Itoa(bn.notificationsCount)) if len(children) > 0 { jChildren, err := json.Marshal(children) if err != nil { return Notification{}, err } body.WriteString(`,"elements":`) body.WriteString(string(jChildren)) } body.WriteString(`}`) notif.Body = body.String() return notif, nil } ================================================ FILE: src/backend/main/go.main/facilities/Notifications/batch_test.go ================================================ package Notifications import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/satori/go.uuid" "github.com/tidwall/gjson" "sync" "testing" ) func TestBatchNotification_aggregate(t *testing.T) { bn := BatchNotification{ emitter: "test", locker: new(sync.Mutex), notificationsCount: 3, notificationsThreshold: notificationsThreshold, notifications: []Notification{ {Body: `{"contact_id": "63ab7904-c416-4f1a-9652-3de82e4fd1f1", "status": "imported"}`, User: &User{UserId: UUID(uuid.FromStringOrNil("63ab7904-c416-4f1a-9652-3de82e4fd1f1"))}}, {Body: `{"contact_id": "63ab7904-c416-4f1a-9652-3de82e4fd1f1", "status": "error", "error_msg": "something went wrong"}`, User: &User{UserId: UUID(uuid.FromStringOrNil("63ab7904-c416-4f1a-9652-3de82e4fd1f1"))}}, {Body: `{"contact_id": "63ab7904-c416-4f1a-9652-3de82e4fd1f1", "status": "ignored"}`, User: &User{UserId: UUID(uuid.FromStringOrNil("63ab7904-c416-4f1a-9652-3de82e4fd1f1"))}}, }, } n, err := bn.aggregate("ref", LongLived) if err != nil { t.Error(err) } else { if !gjson.Valid(n.Body) { t.Error("expected a valid json in body, gjson reported invalid") } else { result := gjson.Get(n.Body, "size") if !result.Exists() { t.Error("expected property 'size' in notification body but gjson can't find it") } else { if result.Int() != 3 { t.Errorf("expected body.size = 3, got %d", result.Int()) } } result = gjson.Get(n.Body, "elements") if !result.Exists() || !result.IsArray() { t.Error("expected body.elements to be an array, gjson reported it doesn't exist or not an array") } else { childrenLength := gjson.Get(n.Body, "elements.#") if childrenLength.Num != 3.0 { t.Errorf("expected an array with 3 obj in body.elements, got %f", childrenLength.Num) } lastChildStatus := gjson.Get(n.Body, "elements.2.body.status") if lastChildStatus.Str != "ignored" { t.Errorf("expected last children.2.body.status == 'ignored', got %s", lastChildStatus.Raw) } } } } bn.notificationsThreshold = 2 n, err = bn.aggregate("", LongLived) if err != nil { t.Error(err) } else { if !gjson.Valid(n.Body) { t.Error("expected a valid json in body, gjson reported invalid") } else { result := gjson.Get(n.Body, "size") if !result.Exists() { t.Error("expected property 'size' in notification body but gjson can't find it") } else { if result.Int() != 3 { t.Errorf("expected body.size = 3, got %d", result.Int()) } } result = gjson.Get(n.Body, "elements") if result.Exists() { t.Error("expected body.elements doesn't exist, but gjson found it") } } } } ================================================ FILE: src/backend/main/go.main/facilities/Notifications/email.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package Notifications import ( "encoding/json" "errors" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "github.com/satori/go.uuid" "time" ) type EmailNotifiers interface { SendEmailAdminToUser(user *User, participants []Participant, email *Message) error } const ( resetPasswordTemplate = "email-reset-password-link.yaml" onboardingEmailTemplate = "email-onboarding.yaml" welcomeEmailTemplate = "email-welcome.yaml" resetLinkFmt = "%s/auth/passwords/reset/%s" deviceValidationLinkFmt = "%s/validate-device/%s" deviceValidationTemplate = "email-device-validation.yaml" ) // ByEmail notifies an user by the mean of an email. func (N *Notifier) ByEmail(notif *Notification) CaliopenError { if N.admin == nil { err := NewCaliopenErr(FailDependencyCaliopenErr, "[NotificationsFacility] can't SendEmailAdminToUser, no admin user has been set") log.Error(err) return err } N.LogNotification("ByEmail", notif) switch notif.Type { case NotifAdminMail: participants := []Participant{ { // sender Address: N.adminLocalID.Identifier, Label: N.adminLocalID.DisplayName, Protocol: EmailProtocol, Type: ParticipantFrom, }, { // recipient Address: notif.User.RecoveryEmail, Contact_ids: []UUID{notif.User.ContactId}, Label: notif.User.Name, Protocol: EmailProtocol, Type: ParticipantTo, }, } if notif.Body == "ccLocalMailbox" { locals, err := N.Store.RetrieveLocalsIdentities(notif.User.UserId.String()) if err != nil { log.WithError(err).Warnf("[SendEmailAdminToUser] failed to retrieve local identities for user %s", notif.User.UserId) } else { for _, localIdentity := range locals { if localIdentity.Identifier != "" { participants = append(participants, Participant{ Address: localIdentity.Identifier, Label: localIdentity.DisplayName, Protocol: EmailProtocol, Type: ParticipantCC, }) } } } } err := N.SendEmailAdminToUser(notif.User, participants, notif.InternalPayload.(*Message)) if err != nil { log.WithError(err).Errorf("[ByEmail] SendEmailAdminToUser failed for notification %+v", *notif) return WrapCaliopenErrf(err, FailDependencyCaliopenErr, "[ByEmail] SendEmailAdminToUser failed") } case NotifPasswordReset: err := N.SendPasswordResetEmail(notif.User, notif.InternalPayload.(*TokenSession)) if err != nil { log.WithError(err).Errorf("[ByEmail] SendPasswordResetEmail failed for notification %+v", *notif) return WrapCaliopenErrf(err, FailDependencyCaliopenErr, "[ByEmail] SendPasswordResetEmail failed") } case OnboardingMails: err := N.SendOnboardingMails(notif.User) if err != nil { log.WithError(err).Errorf("[ByEmail] SendOnboardingMails failed for notification %+v", *notif) return WrapCaliopenErrf(err, FailDependencyCaliopenErr, "[ByEmail] SendOnboardingMails failed") } case NotifDeviceValidation: err := N.SendDeviceValidationEmail(notif.User, notif.Body, notif.InternalPayload.(*TokenSession)) if err != nil { log.WithError(err).Errorf("[ByEmail] SendDeviceValidationEmail failed for notification %+v", *notif) return WrapCaliopenErrf(err, FailDependencyCaliopenErr, "[ByEmail] SendDeviceValidationEmail failed") } default: log.Errorf("[Notifier]ByEmail : unknown notification type <%s>", notif.Type) return NewCaliopenErrf(UnprocessableCaliopenErr, "[Notifier]ByEmail : unknown notification type <%s>", notif.Type) } return nil } // SendEmailAdminToUser sends an administrative email to recipients found in participants slice. // Participants must include at least one `From` and one `To` // this is an email composed by the backend to inform user that something happened related to its account // func is in charge of saving & indexing draft before sending the "deliver" order to the SMTP broker. func (notif *Notifier) SendEmailAdminToUser(user *User, participants []Participant, email *Message) error { if email == nil { return errors.New("[SendEmailAdminToUser] can't send a nil email") } now := time.Now() (*email).Date = now (*email).Date_insert = now (*email).Message_id.UnmarshalBinary(uuid.NewV4().Bytes()) //TODO : (*email).Discussion_id.UnmarshalBinary(uuid.NewV4().Bytes()) (*email).Is_draft = true (*email).Participants = participants (*email).Protocol = EmailProtocol (*email).User_id = notif.admin.UserId (*email).UserIdentities = []UUID{notif.adminLocalID.Id} // save & index message err := notif.Store.CreateMessage(email) if err != nil { log.WithError(err).Warn("[EmailNotifiers]: SendEmailAdminToUser failed to store draft") return err } user_info := &UserInfo{User_id: notif.admin.UserId.String(), Shard_id: notif.admin.ShardId} err = notif.index.CreateMessage(user_info, email) if err != nil { log.WithError(err).Warn("[EmailNotifiers]: SendEmailAdminToUser failed to index draft") return err } log.Infof("[NotificationsFacility] sending email admin for user <%s> [%s]", user.Name, user.UserId.String()) const nats_order = "deliver" order := BrokerOrder{ Order: nats_order, MessageId: email.Message_id.String(), UserId: notif.admin.UserId.String(), IdentityId: notif.adminLocalID.Id.String(), } natsMessage, e := json.Marshal(order) if e != nil { return fmt.Errorf("[EmailNotifiers] failed to build nats message : %s", e.Error()) } rep, err := notif.NatsQueue.Request(notif.natsTopics[Nats_outSMTP_topicKey], natsMessage, 30*time.Second) if err != nil { log.WithError(err).Warn("[EmailNotifiers]: SendEmailAdminToUser error") if notif.NatsQueue.LastError() != nil { log.WithError(notif.NatsQueue.LastError()).Warn("[EmailNotifiers]: SendEmailAdminToUser error") return err } return err } var reply DeliveryAck err = json.Unmarshal(rep.Data, &reply) if err != nil { log.WithError(err).Warn("[EmailNotifiers]: SendEmailAdminToUser error") return err } if reply.Err { err := errors.New(reply.Response) log.WithError(err).Warn("[EmailNotifiers]: SendEmailAdminToUser error") return err } return nil } func (notif *Notifier) SendPasswordResetEmail(user *User, session *TokenSession) error { if user == nil || session == nil { return errors.New("[NotificationsFacility] SendPasswordResetEmail invalid params") } reset_link := fmt.Sprintf(resetLinkFmt, notif.config.BaseUrl, session.Token) context := map[string]interface{}{ "given_name": user.GivenName, "family_name": user.FamilyName, "domain": notif.config.BaseUrl, "url": reset_link, } email, err := RenderEmail(notif.config.TemplatesPath+resetPasswordTemplate, context) if err != nil { log.WithError(err).Warnf("[RESTfacility] failed to build reset email from template for user %s", user.UserId.String()) return errors.New("[RESTfacility] failed to build reset email") } participants := []Participant{ { // sender Address: (*notif.adminLocalID).Identifier, Label: (*notif.adminLocalID).DisplayName, Protocol: EmailProtocol, Type: ParticipantFrom, }, { // recipient Address: user.RecoveryEmail, Contact_ids: []UUID{user.ContactId}, Label: user.Name, Protocol: EmailProtocol, Type: ParticipantTo, }, } err = notif.SendEmailAdminToUser(user, participants, email) if err != nil { log.WithError(err).Warnf("[RESTfacility] sending password reset email failed for user %s", user.UserId.String()) return errors.New("[RESTfacility] failed to send password reset email") } return nil } // SendOnboardingMails builds and sends one-time emails to user that signed-up func (notif *Notifier) SendOnboardingMails(user *User) error { if user == nil { return errors.New("[SendOnboardingMails] must be called with an user, got nil") } recipients := []Participant{} // retrieve user's local email locals, err := notif.Store.RetrieveLocalsIdentities(user.UserId.String()) if err != nil { log.WithError(err).Warnf("[SendEmailAdminToUser] failed to retrieve local identities for user %s", user.UserId) return errors.New("failed to retrieve user's local email address, can't send email") } for _, localIdentity := range locals { if localIdentity.Identifier != "" { recipients = append(recipients, Participant{ Address: localIdentity.Identifier, Label: localIdentity.DisplayName, Protocol: EmailProtocol, Type: ParticipantTo, }) } } // build and send first email onboardingMail, err := RenderEmail(notif.config.TemplatesPath+onboardingEmailTemplate, map[string]interface{}{}) if err == nil { participants := recipients participants = append(participants, Participant{ // sender Address: "contact@caliopen.org", // TODO: use config Label: "Caliopen", Protocol: EmailProtocol, Type: ParticipantFrom, }) err = notif.SendEmailAdminToUser(user, participants, onboardingMail) if err != nil { log.WithError(err).Warnf("[SendOnboardingMails] failed to send onboarding mail for user %s", user.UserId) } } else { log.WithError(err).Warnf("[SendOnboardingMails] failed to render onboardingMail. This email won't be send.") } // build and send first email welcomeMail, err := RenderEmail(notif.config.TemplatesPath+welcomeEmailTemplate, map[string]interface{}{}) if err == nil { participants := recipients participants = append(participants, Participant{ // sender Address: "contact@caliopen.org", // TODO: use config Label: "Laurent Chemla", Protocol: EmailProtocol, Type: ParticipantFrom, }) err = notif.SendEmailAdminToUser(user, participants, welcomeMail) if err != nil { log.WithError(err).Warnf("[SendOnboardingMails] failed to send welcome mail for user %s", user.UserId) } } else { log.WithError(err).Warnf("[SendOnboardingMails] failed to render welcomeMail. This email won't be send.") } return nil } func (notif *Notifier) SendDeviceValidationEmail(user *User, deviceName string, session *TokenSession) error { if user == nil || session == nil { return errors.New("[NotificationsFacility] SendDeviceValidationEmail invalid params") } reset_link := fmt.Sprintf(deviceValidationLinkFmt, notif.config.BaseUrl, session.Token) context := map[string]interface{}{ "given_name": user.GivenName, "family_name": user.FamilyName, "device_name": deviceName, "url": reset_link, } email, err := RenderEmail(notif.config.TemplatesPath+deviceValidationTemplate, context) if err != nil { log.WithError(err).Warnf("[SendDeviceValidationEmail] failed to build device validation email from template for user %s", user.UserId.String()) return errors.New("[SendDeviceValidationEmail] failed to build validation email") } participants := []Participant{ { // sender Address: (*notif.adminLocalID).Identifier, Label: (*notif.adminLocalID).DisplayName, Protocol: EmailProtocol, Type: ParticipantFrom, }, { // recipient Address: user.RecoveryEmail, Contact_ids: []UUID{user.ContactId}, Label: user.Name, Protocol: EmailProtocol, Type: ParticipantTo, }, } err = notif.SendEmailAdminToUser(user, participants, email) if err != nil { log.WithError(err).Warnf("[SendDeviceValidationEmail] sending device validation email failed for user %s", user.UserId.String()) return errors.New("[SendDeviceValidationEmail] failed to send device validation email") } return nil } ================================================ FILE: src/backend/main/go.main/facilities/Notifications/facility.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package Notifications import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/index/elasticsearch" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/cassandra" log "github.com/Sirupsen/logrus" "github.com/gocql/gocql" "github.com/nats-io/go-nats" "os" "time" ) type ( Notifiers interface { ByEmail(*Notification) CaliopenError ByNotifQueue(*Notification) CaliopenError NotificationsByTime(userId string, from, to time.Time) ([]Notification, CaliopenError) NotificationsByID(userId, from, to string) ([]Notification, CaliopenError) RetrieveNotification(userId, notificationId string) (Notification, CaliopenError) DeleteNotifications(userId string, until time.Time) CaliopenError DeleteNotification(userId, notificationId string) CaliopenError } Notifier struct { admin *User // Admin user on whose behalf actions could be done adminLocalID *UserIdentity // Admin's local identity used to send emails config *NotifierConfig index backends.NotificationsIndex NatsQueue *nats.Conn natsTopics map[string]string Store backends.NotificationsStore log *log.Logger } ) // NewNotificationsFacility initialises the notifiers // it takes the same store & index configurations than the REST API for now func NewNotificationsFacility(config CaliopenConfig, queue *nats.Conn) (notifier *Notifier) { notifier = new(Notifier) notifier.log = log.New() notifier.log.Out = os.Stdout // We could set this to any `io.Writer` such as a file // file, err := os.OpenFile("notifications.log", os.O_CREATE|os.O_WRONLY, 0666) // if err == nil { // notifier.log.Out = file // } else { // log.Info("Failed to log to file, using default stdout") // notifier.log.Out = os.Stdout // } notifier.config = &config.NotifierConfig notifier.natsTopics = make(map[string]string) notifier.natsTopics[Nats_outSMTP_topicKey] = config.NatsConfig.OutSMTP_topic notifier.natsTopics[Nats_Contacts_topicKey] = config.NatsConfig.Contacts_topic notifier.NatsQueue = queue switch config.RESTstoreConfig.BackendName { case "cassandra": cassaConfig := store.CassandraConfig{ Hosts: config.RESTstoreConfig.Hosts, Keyspace: config.RESTstoreConfig.Keyspace, Consistency: gocql.Consistency(config.RESTstoreConfig.Consistency), } if config.RESTstoreConfig.ObjStoreType == "s3" { cassaConfig.WithObjStore = true cassaConfig.OSSConfig.Endpoint = config.RESTstoreConfig.OSSConfig.Endpoint cassaConfig.OSSConfig.AccessKey = config.RESTstoreConfig.OSSConfig.AccessKey cassaConfig.OSSConfig.SecretKey = config.RESTstoreConfig.OSSConfig.SecretKey cassaConfig.OSSConfig.Location = config.RESTstoreConfig.OSSConfig.Location cassaConfig.OSSConfig.RawMsgBucket = config.RESTstoreConfig.OSSConfig.Buckets["raw_messages"] cassaConfig.OSSConfig.AttachmentBucket = config.RESTstoreConfig.OSSConfig.Buckets["temporary_attachments"] } backend, err := store.InitializeCassandraBackend(cassaConfig) if err != nil { log.WithError(err).Fatalf("Initalization of %s backend failed", config.RESTstoreConfig.BackendName) } notifier.Store = backends.NotificationsStore(backend) // type conversion default: log.Fatalf("Unknown backend: %s", config.RESTstoreConfig.BackendName) } switch config.RESTindexConfig.IndexName { case "elasticsearch": esConfig := index.ElasticSearchConfig{ Urls: config.RESTindexConfig.Hosts, } index, err := index.InitializeElasticSearchIndex(esConfig) if err != nil { log.WithError(err).Fatalf("Initalization of %s index failed", config.RESTindexConfig.IndexName) } notifier.index = backends.NotificationsIndex(index) // type conversion default: log.Fatalf("Unknown index: %s", config.RESTindexConfig.IndexName) } user, err := notifier.Store.UserByUsername(config.NotifierConfig.AdminUsername) if err != nil { log.WithError(err).Warnf("Failed to retrieve admin user <%s>", config.NotifierConfig.AdminUsername) } else if user != nil { notifier.admin = user ids, err := notifier.Store.RetrieveLocalsIdentities(user.UserId.String()) if err != nil { log.WithError(err).Warnf("Failed to retrieve local identities for admin user <%s>", config.NotifierConfig.AdminUsername) } else { // get first local identity found for now notifier.adminLocalID = &(ids[0]) } } return notifier } func (N *Notifier) LogNotification(method string, notif *Notification) { if notif != nil { var userId string if notif.User != nil { userId = notif.User.UserId.String() } else { userId = "" } N.log.WithFields(log.Fields{ "method": method, "notif_id": notif.NotifId.String(), }).Infof("[Notifier] a notification has been issued for user %s", userId) } } func (N *Notifier) NotificationsByTime(userId string, from, to time.Time) ([]Notification, CaliopenError) { notifs, err := N.Store.NotificationsByTime(userId, from, to) if err != nil { return []Notification{}, WrapCaliopenErr(err, DbCaliopenErr, "[RetrieveNotifications] failed") } return notifs, nil } func (N *Notifier) NotificationsByID(userId, from, to string) ([]Notification, CaliopenError) { notifs, err := N.Store.NotificationsByID(userId, from, to) if err != nil { return []Notification{}, WrapCaliopenErr(err, DbCaliopenErr, "[RetrieveNotifications] failed") } return notifs, nil } func (N *Notifier) RetrieveNotification(userID, notifID string) (Notification, CaliopenError) { notif, err := N.Store.RetrieveNotification(userID, notifID) if err != nil { return Notification{}, WrapCaliopenErr(err, DbCaliopenErr, "[RetrieveNotification] failed") } return notif, nil } func (N *Notifier) DeleteNotifications(userId string, until time.Time) CaliopenError { err := N.Store.DeleteNotifications(userId, until) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[Notifier]DeleteNotifications failed") } return nil } func (N *Notifier) DeleteNotification(userID, notifID string) CaliopenError { err := N.Store.DeleteNotification(userID, notifID) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[Notifier]DeleteNotifications failed") } return nil } ================================================ FILE: src/backend/main/go.main/facilities/Notifications/queue.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package Notifications import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/Sirupsen/logrus" ) // ByNotifQueue notifies an user by the mean of the notification queue func (N *Notifier) ByNotifQueue(notif *Notification) CaliopenError { N.LogNotification("ByNotificationQueue", notif) err := N.Store.PutNotificationInQueue(notif) if err != nil { logrus.WithError(err).Errorf("[Notifier]ByNotifQueue failed to put notification in queue") return WrapCaliopenErr(err, DbCaliopenErr, "[Notifier]ByNotifQueue failed to put notification in queue") } return nil } ================================================ FILE: src/backend/main/go.main/facilities/Notifications/templating.go ================================================ package Notifications import ( "encoding/base64" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "gopkg.in/flosch/pongo2.v3" "gopkg.in/yaml.v2" "io/ioutil" "path" ) var templateDir string // RenderResetEmail will load the yaml/j2 template from template_path // and scaffold a Message with data provided in context map func RenderEmail(template_path string, context map[string]interface{}) (*Message, error) { templateDir = path.Dir(template_path) // adds our base64encodeImgSrc encoder in context context["base64encodeImgSrc"] = base64encodeImgSrc // load email attributes from template file var email map[string]interface{} file, err := ioutil.ReadFile(template_path) if err != nil { return nil, err } err = yaml.Unmarshal(file, &email) if err != nil { return nil, err } var bodyHtml, bodyPlain, subject string if s, found := email["subject"]; found { if s_, ok := s.(string); ok && s_ != "" { tpl, err := pongo2.FromString(s_) if err == nil { subject, _ = tpl.Execute(pongo2.Context(context)) } else { log.WithError(err).Warnf("[RenderEmail] failed to execute template %s", template_path) } } } if plain, found := email["body_plain"]; found { if p, ok := plain.(string); ok && p != "" { tpl, err := pongo2.FromString(p) if err == nil { bodyPlain, err = tpl.Execute(pongo2.Context(context)) } else { log.WithError(err).Warnf("[RenderEmail] failed to execute template %s", template_path) } } } if html, found := email["body_html"]; found { if h, ok := html.(string); ok && h != "" { tpl, err := pongo2.FromString(h) if err == nil { bodyHtml, err = tpl.Execute(pongo2.Context(context)) } else { log.WithError(err).Warnf("[RenderEmail] failed to execute template %s", template_path) } } } // create Message with strings return &Message{ Subject: subject, Body_html: bodyHtml, Body_plain: bodyPlain, }, nil } // base64encodeImgSrc takes a path to a file and returns src string to inline image in base64 // it implements pongo2 filter interface func base64encodeImgSrc(in *pongo2.Value) *pongo2.Value { imgPath := templateDir + "/" + in.String() if len(imgPath) < 6 { return pongo2.AsValue("") } imgExt := path.Ext(imgPath)[1:] img, err := ioutil.ReadFile(imgPath) if err != nil { return pongo2.AsValue(err.Error()) } base64Img := base64.StdEncoding.EncodeToString(img) return pongo2.AsValue(`data:image/` + imgExt + `;base64,` + base64Img) } ================================================ FILE: src/backend/main/go.main/facilities/REST/RESTfacility.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package REST import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/cache" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/index/elasticsearch" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/cassandra" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/Notifications" log "github.com/Sirupsen/logrus" "github.com/gocql/gocql" "github.com/nats-io/go-nats" "github.com/tidwall/gjson" "io" ) type ( RESTservices interface { UsernameIsAvailable(string) (bool, error) SuggestRecipients(user *UserInfo, query_string string) (suggests []RecipientSuggestion, err error) GetSettings(user_id string) (settings *Settings, err error) //contacts CreateContact(user *UserInfo, contact *Contact) error RetrieveContacts(filter IndexSearch) (contacts []*Contact, totalFound int64, err error) RetrieveContact(userID, contactID string) (*Contact, error) RetrieveUserContact(userID string) (*Contact, error) UpdateContact(user *UserInfo, contact, oldContact *Contact, update map[string]interface{}) error PatchContact(user *UserInfo, patch []byte, contactID string) error DeleteContact(user *UserInfo, contactID string) error ContactExists(userID, contactID string) bool LookupContactByUri(userID, uri string) ([]*Contact, int64, error) ImportVcardFile(info *UserInfo, file io.Reader) error //identities RetrieveContactIdentities(user_id, contact_id string) (identities []ContactIdentity, err error) RetrieveLocalIdentities(user_id string) (identities []UserIdentity, err error) CreateUserIdentity(identity *UserIdentity) CaliopenError RetrieveRemoteIdentities(userId string, withCredentials bool) (ids []*UserIdentity, err CaliopenError) RetrieveUserIdentity(userId, RemoteId string, withCredentials bool) (id *UserIdentity, err CaliopenError) UpdateUserIdentity(identity, oldIdentity *UserIdentity, update map[string]interface{}) CaliopenError PatchUserIdentity(patch []byte, userId, RemoteId string) CaliopenError DeleteUserIdentity(userId, remoteId string) CaliopenError IsRemoteIdentity(userId, remoteId string) bool //providers RetrieveProvidersList() (providers []Provider, err error) GetProviderOauthFor(userID, provider, instance string) (Provider, CaliopenError) CreateTwitterIdentity(requestToken, verifier string) (remoteId string, err CaliopenError) CreateGmailIdentity(state, code string) (remoteId string, err CaliopenError) CreateMastodonIdentity(state, code string) (remoteId string, err CaliopenError) //discussions GetDiscussionsList(user *UserInfo, ILrange, PIrange [2]int8, limit, offset int) ([]Discussion, int, error) DiscussionMetadata(user *UserInfo, discussionId string) (Discussion, error) //messages GetMessagesList(filter IndexSearch) (messages []*Message, totalFound int64, err error) GetMessagesRange(filter IndexSearch) (messages []*Message, totalFound int64, err error) GetMessage(user *UserInfo, message_id string) (message *Message, err error) SendDraft(user *UserInfo, msg_id string) (msg *Message, err error) SetMessageUnread(user *UserInfo, message_id string, status bool) error GetRawMessage(raw_message_id string) (message []byte, err error) //attachments AddAttachment(user *UserInfo, message_id, filename, content_type string, file io.Reader) (attachmentURL string, err error) DeleteAttachment(user *UserInfo, message_id string, attchmt_id string) CaliopenError OpenAttachment(user_id, message_id string, attchmtIndex string) (meta map[string]string, content io.Reader, err error) //tags RetrieveUserTags(user_id string) (tags []Tag, err CaliopenError) CreateTag(tag *Tag) CaliopenError RetrieveTag(user_id, tag_id string) (tag Tag, err CaliopenError) UpdateTag(tag *Tag) CaliopenError PatchTag(patch []byte, user_id, tag_name string) CaliopenError DeleteTag(user_id, tag_name string) CaliopenError UpdateResourceTags(user *UserInfo, resourceID, resourceType string, patch []byte) CaliopenError //search Search(IndexSearch) (result *IndexResult, err error) //users PatchUser(user_id string, patch *gjson.Result, notifier Notifications.Notifiers) error RequestPasswordReset(payload PasswordResetRequest, notifier Notifications.Notifiers) error ValidatePasswordResetToken(token string) (session *TokenSession, err error) ResetUserPassword(token, new_password string, notifier Notifications.Notifiers) error DeleteUser(payload ActionsPayload) CaliopenError //devices CreateDevice(device *Device) CaliopenError RetrieveDevices(userId string) ([]Device, CaliopenError) RetrieveDevice(userId, deviceId string) (*Device, CaliopenError) UpdateDevice(device, oldDevice *Device, update map[string]interface{}) CaliopenError PatchDevice(patch []byte, userId, deviceId string) CaliopenError DeleteDevice(userId, deviceId string) CaliopenError RequestDeviceValidation(userId, deviceId, channel string, notifier Notifications.Notifiers) CaliopenError ConfirmDeviceValidation(userId, token string) CaliopenError //keys CreatePGPPubKey(label string, pubkey []byte, contact *Contact) (*PublicKey, CaliopenError) RetrieveContactPubKeys(userId, contactId string) (pubkeys PublicKeys, err CaliopenError) RetrievePubKey(userId, resourceId, keyId string) (pubkey *PublicKey, err CaliopenError) DeletePubKey(pubkey *PublicKey) CaliopenError PatchPubKey(patch []byte, userId, resourceId, keyId string) CaliopenError } RESTfacility struct { Cache backends.APICache index backends.APIIndex natsTopics map[string]string nats_conn *nats.Conn providers map[string]Provider store backends.APIStorage Hostname string } ) func NewRESTfacility(config CaliopenConfig, nats_conn *nats.Conn) (rest_facility *RESTfacility) { rest_facility = new(RESTfacility) rest_facility.nats_conn = nats_conn rest_facility.natsTopics = map[string]string{ Nats_outSMTP_topicKey: config.NatsConfig.OutSMTP_topic, Nats_outIMAP_topicKey: config.NatsConfig.OutIMAP_topic, Nats_Contacts_topicKey: config.NatsConfig.Contacts_topic, Nats_outTwitter_topicKey: config.NatsConfig.OutTWITTER_topic, Nats_outMastodon_topicKey: config.NatsConfig.OutMASTODON_topic, Nats_Keys_topicKey: config.NatsConfig.Keys_topic, Nats_IdPoller_topicKey: config.NatsConfig.IdPoller_topic, } switch config.RESTstoreConfig.BackendName { case "cassandra": cassaConfig := store.CassandraConfig{ Hosts: config.RESTstoreConfig.Hosts, Keyspace: config.RESTstoreConfig.Keyspace, Consistency: gocql.Consistency(config.RESTstoreConfig.Consistency), UseVault: config.RESTstoreConfig.UseVault, } if config.RESTstoreConfig.ObjStoreType == "s3" { cassaConfig.WithObjStore = true cassaConfig.OSSConfig.Endpoint = config.RESTstoreConfig.OSSConfig.Endpoint cassaConfig.OSSConfig.AccessKey = config.RESTstoreConfig.OSSConfig.AccessKey cassaConfig.OSSConfig.SecretKey = config.RESTstoreConfig.OSSConfig.SecretKey cassaConfig.OSSConfig.Location = config.RESTstoreConfig.OSSConfig.Location cassaConfig.OSSConfig.RawMsgBucket = config.RESTstoreConfig.OSSConfig.Buckets["raw_messages"] cassaConfig.OSSConfig.AttachmentBucket = config.RESTstoreConfig.OSSConfig.Buckets["temporary_attachments"] } if config.RESTstoreConfig.UseVault { cassaConfig.HVaultConfig.Url = config.RESTstoreConfig.VaultConfig.Url cassaConfig.HVaultConfig.Username = config.RESTstoreConfig.VaultConfig.Username cassaConfig.HVaultConfig.Password = config.RESTstoreConfig.VaultConfig.Password } backend, err := store.InitializeCassandraBackend(cassaConfig) if err != nil { log.WithError(err).Fatalf("initalization of %s backend failed", config.RESTstoreConfig.BackendName) } rest_facility.store = backends.APIStorage(backend) // type conversion default: log.Fatalf("unknown backend: %s", config.RESTstoreConfig.BackendName) } switch config.RESTindexConfig.IndexName { case "elasticsearch": esConfig := index.ElasticSearchConfig{ Urls: config.RESTindexConfig.Hosts, } indx, err := index.InitializeElasticSearchIndex(esConfig) if err != nil { log.WithError(err).Fatalf("initalization of %s index failed", config.RESTindexConfig.IndexName) } rest_facility.index = backends.APIIndex(indx) // type conversion default: log.Fatalf("unknown index: %s", config.RESTindexConfig.IndexName) } var err error rest_facility.Cache, err = cache.InitializeRedisBackend(config.CacheConfig) if err != nil { log.WithError(err).Fatal("initialization of Redis cache failed") } rest_facility.providers = map[string]Provider{} for _, provider := range config.Providers { if provider.Name != "" { rest_facility.providers[provider.Name] = provider } } rest_facility.Hostname = config.Hostname return rest_facility } ================================================ FILE: src/backend/main/go.main/facilities/REST/attachment.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package REST import ( "bytes" "errors" "fmt" "github.com/CaliOpen/Caliopen/src/backend/brokers/go.emails" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/satori/go.uuid" "io" "strconv" ) func (rest *RESTfacility) AddAttachment(user *UserInfo, message_id, filename, content_type string, file io.Reader) (tempId string, err error) { //check if message_id belongs to user and is a draft msg, err := rest.store.RetrieveMessage(user.User_id, message_id) if err != nil { return "", err } if !msg.Is_draft { return "", errors.New("message " + message_id + " is not a draft.") } //store temporary file in objectStore facility tmpId := uuid.NewV4() tempId = tmpId.String() url, size, err := rest.store.StoreAttachment(tempId, file) if err != nil { return "", err } //update draft with new attachment references draftAttchmnt := Attachment{ ContentType: content_type, FileName: filename, IsInline: false, Size: size, TempID: UUID(tmpId), URL: url, } draftAttchmnt.TempID.UnmarshalBinary(tmpId.Bytes()) msg.Attachments = append(msg.Attachments, draftAttchmnt) //update store fields := make(map[string]interface{}) fields["Attachments"] = msg.Attachments err = rest.store.UpdateMessage(msg, fields) if err != nil { //roll-back attachment storage before returning the error rest.store.DeleteAttachment(url) return "", err } //update index err = rest.index.UpdateMessage(user, msg, fields) if err != nil { //roll-back attachment storage before returning the error fields["Attachments"] = msg.Attachments[:len(msg.Attachments)-1] rest.store.UpdateMessage(msg, fields) rest.store.DeleteAttachment(url) return "", err } return } func (rest *RESTfacility) DeleteAttachment(user *UserInfo, message_id string, attchmt_id string) CaliopenError { //check if message_id belongs to user and is a draft and index is consistent msg, err := rest.store.RetrieveMessage(user.User_id, message_id) if err != nil { var msg string if err.Error() == "not found" { msg = "message not found" } return WrapCaliopenErr(err, DbCaliopenErr, msg) } if !msg.Is_draft { return NewCaliopenErrf(ForbiddenCaliopenErr, "message %s is not a draft", message_id) } //find and remove attachment's from draft for i, attachment := range msg.Attachments { if attachment.TempID.String() == attchmt_id { msg.Attachments = append(msg.Attachments[:i], msg.Attachments[i+1:]...) //update store fields := make(map[string]interface{}) fields["Attachments"] = msg.Attachments rest.store.UpdateMessage(msg, fields) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "") } //update index err = rest.index.UpdateMessage(user, msg, fields) //remove temporary file from object store err = rest.store.DeleteAttachment(attachment.URL) if err != nil { return WrapCaliopenErrf(err, DbCaliopenErr, "failed to remove temp attachment at uri '%s' with error <%s>", attachment.URL, err.Error()) } return nil } } return NewCaliopenErr(NotFoundCaliopenErr, "attachment not found") } // returns an io.Reader and metadata to conveniently read the attachment func (rest *RESTfacility) OpenAttachment(user_id, message_id, attchmtIndex string) (meta map[string]string, content io.Reader, err error) { if attchmtIndex == "" { return meta, nil, errors.New(fmt.Sprint("empty attachment id")) } meta = make(map[string]string) //check if message_id belongs to user and index is consistent msg, err := rest.store.RetrieveMessage(user_id, message_id) if err != nil { return meta, nil, err } var index int if msg.Is_draft { // retrieve attachment by temp_id notfound := true for _, att := range msg.Attachments { if att.TempID.String() == attchmtIndex { meta["Content-Type"] = att.ContentType meta["Message-Size"] = strconv.Itoa(att.Size) meta["Filename"] = att.FileName meta["Url"] = att.URL notfound = false break } } if notfound { return meta, nil, NewCaliopenErr(NotFoundCaliopenErr, "attachment not found") } } else { // retrieve attachment by index index, err = strconv.Atoi(attchmtIndex) if err != nil || index < 0 || index > (len(msg.Attachments)-1) { return meta, nil, NewCaliopenErr(NotFoundCaliopenErr, "attachment not found") } meta["Content-Type"] = msg.Attachments[index].ContentType meta["Message-Size"] = strconv.Itoa(msg.Attachments[index].Size) meta["Filename"] = msg.Attachments[index].FileName } // create a Reader // either from object store (draft context) // or from raw message's mime part (non-draft context) if msg.Is_draft { attachment, e := rest.store.GetAttachment(meta["Url"]) if e != nil { return map[string]string{}, nil, e } content = attachment return } else { rawMsg, e := rest.store.GetRawMessage(msg.Raw_msg_id.String()) if e != nil { return map[string]string{}, nil, e } json_email, e := email_broker.EmailToJsonRep(rawMsg.Raw_data) if e != nil { return map[string]string{}, nil, e } attachments, e := json_email.ExtractAttachments(index) if e != nil { return map[string]string{}, nil, e } content = bytes.NewReader(attachments[0]) return } } ================================================ FILE: src/backend/main/go.main/facilities/REST/contacts.go ================================================ /* * // Copyleft (ɔ) 2017 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package REST import ( "encoding/json" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/contact" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/helpers" log "github.com/Sirupsen/logrus" "github.com/bitly/go-simplejson" "github.com/pkg/errors" "github.com/satori/go.uuid" "io" "strings" "time" ) // CreateContact validates Contact before saving it to cassandra and ES func (rest *RESTfacility) CreateContact(user *UserInfo, contact *Contact) (err error) { // add missing properties contact.ContactId.UnmarshalBinary(uuid.NewV4().Bytes()) contact.DateInsert = time.Now() contact.DateUpdate = contact.DateInsert // normalization helpers.ComputeNewTitle(contact) helpers.NormalizePhoneNumbers(contact) MarshalNested(contact) MarshalRelated(contact) err = rest.store.CreateContact(contact) if err != nil { return err } err = rest.index.CreateContact(user, contact) if err != nil { return err } // notify external components go func(contact *Contact) { const updatePI_order = "contact_update" natsMessage := fmt.Sprintf(Nats_contact_tmpl, updatePI_order, contact.ContactId.String(), contact.UserId.String()) rest.PublishOnNats(natsMessage, rest.natsTopics[Nats_Contacts_topicKey]) }(contact) return nil } // RetrieveContacts returns contacts collection from index given filter params func (rest *RESTfacility) RetrieveContacts(filter IndexSearch) (contacts []*Contact, totalFound int64, err error) { contacts, totalFound, err = rest.index.FilterContacts(filter) if err != nil { return []*Contact{}, 0, err } return } // RetrieveContact returns one contact func (rest *RESTfacility) RetrieveContact(userID, contactID string) (contact *Contact, err error) { return rest.store.RetrieveContact(userID, contactID) } func (rest *RESTfacility) LookupContactByUri(userID, uri string) (contacts []*Contact, totalFound int64, err error) { contacts = []*Contact{} uri = strings.ToLower(uri) uriSplit := strings.SplitN(uri, ":", 2) if len(uriSplit) != 2 { err = fmt.Errorf("[LookupContactByUri] uri malformed : %s => %v", uri, uriSplit) log.WithError(err) return } ids, err := rest.store.LookupContactsByIdentifier(userID, uriSplit[1], uriSplit[0]) if err != nil { return } for _, contactId := range ids { c := Contact{} c.ContactId = UUID(uuid.FromStringOrNil(contactId)) contacts = append(contacts, &c) totalFound++ } return } // RetrieveUserContact returns the contact entry belonging to user. // This is the contact that is auto-created for user and can't be deleted. func (rest *RESTfacility) RetrieveUserContact(userID string) (contact *Contact, err error) { contactID := rest.store.RetrieveUserContactId(userID) if contactID == "" { return nil, NewCaliopenErrf(NotFoundCaliopenErr, "[RetrieveUserContact] didn't find contact id for user %s", userID) } contact, err = rest.store.RetrieveContact(userID, contactID) if err != nil { return nil, err } return } // PatchContact is a shortcut for REST api to : // - retrieve the contact from db // - UpdateWithPatch() with UserActor role // - then UpdateContact() to save updated contact to stores & index if everything went good. func (rest *RESTfacility) PatchContact(user *UserInfo, patch []byte, contactID string) error { // TODO : fix removing identities which fails silently on user's contact current_contact, err := rest.RetrieveContact(user.User_id, contactID) if err != nil { if err.Error() == "not found" { return NewCaliopenErr(NotFoundCaliopenErr, "[RESTfacility] contact not found") } else { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] PatchContact failed to retrieve contact") } } // read into the patch to make basic controls before processing it with generic helper patchReader, err := simplejson.NewJson(patch) if err != nil { return WrapCaliopenErrf(err, FailDependencyCaliopenErr, "[RESTfacility] PatchContact failed with simplejson error : %s", err) } // forbid tags modification immediately if _, hasTagsProp := patchReader.CheckGet("tags"); hasTagsProp { return NewCaliopenErr(ForbiddenCaliopenErr, "[RESTfacility] PatchContact : patching tags through parent object is forbidden") } // checks "current_state" property is present if _, hasCurrentState := patchReader.CheckGet("current_state"); !hasCurrentState { return NewCaliopenErr(ForbiddenCaliopenErr, "[RESTfacility] PatchContact : current_state property must be in patch") } // patch seams OK, apply it to the resource var modifiedFields map[string]interface{} newContact, modifiedFields, err := helpers.UpdateWithPatch(patch, current_contact, UserActor) if err != nil { log.WithError(err).Warn("[RESTfacility] PatchContact failed") return WrapCaliopenErr(err, FailDependencyCaliopenErr, "[RESTfacility] PatchContact failed") } needNewTitle := false discoverKey := false for key := range modifiedFields { switch key { // check if title has to be re-computed case "AdditionalName", "FamilyName", "GivenName", "NamePrefix", "NameSuffix": needNewTitle = true // Check if we can try to discover a public key case "Emails", "Identities": discoverKey = true } } if needNewTitle { helpers.ComputeTitle(newContact.(*Contact)) modifiedFields["Title"] = newContact.(*Contact).Title } // save updated resource err = rest.UpdateContact(user, newContact.(*Contact), current_contact, modifiedFields) if err != nil { if strings.HasPrefix(err.Error(), "uri <") { return WrapCaliopenErrf(err, ForbiddenCaliopenErr, "[RESTfacility] PatchContact forbidden : %s", err) } else { return WrapCaliopenErrf(err, FailDependencyCaliopenErr, "[RESTfacility] PatchContact failed with UpdateContact error : %s", err) } } if discoverKey { err = rest.launchKeyDiscovery(current_contact, modifiedFields) } return nil } func (rest *RESTfacility) launchKeyDiscovery(current_contact *Contact, updatedFields map[string]interface{}) error { go func(contact *Contact) { const discover_order = "discover_key" message := DiscoverKeyMessage{Order: discover_order, ContactId: current_contact.ContactId.String(), UserId: current_contact.UserId.String()} if value, ok := updatedFields["Emails"]; ok { message.Emails = value.([]EmailContact) } if value, ok := updatedFields["Identities"]; ok { message.Identities = value.([]SocialIdentity) } natsMessage, err := json.Marshal(message) if err != nil { return } log.Infof("Will publish nats topic %s for contact %s", rest.natsTopics[Nats_Keys_topicKey], current_contact.ContactId.String()) rest.PublishOnNats(string(natsMessage), rest.natsTopics[Nats_Keys_topicKey]) }(current_contact) return nil } // UpdateContact updates a contact in store & index with payload func (rest *RESTfacility) UpdateContact(user *UserInfo, contact, oldContact *Contact, modifiedFields map[string]interface{}) error { err := rest.store.UpdateContact(contact, oldContact, modifiedFields) if err != nil { return err } err = rest.index.UpdateContact(user, contact, modifiedFields) if err != nil { return err } // notify external components go func(contact *Contact) { const update_order = "contact_update" natsMessage := fmt.Sprintf(Nats_contact_tmpl, update_order, contact.ContactId.String(), contact.UserId.String()) err := rest.PublishOnNats(natsMessage, rest.natsTopics[Nats_Contacts_topicKey]) if err != nil { log.WithError(err).Error("[UpdateContact] failed to publish contact_update on nats") } }(contact) return nil } // DeleteContact deletes a contact in store & index, only if : // - contact belongs to user ;-) // - contact is not the user's contact card func (rest *RESTfacility) DeleteContact(info *UserInfo, contactID string) error { user, err := rest.store.RetrieveUser(info.User_id) if err != nil { return err } c, err := rest.store.RetrieveContact(info.User_id, contactID) if err != nil { return err } if user.ContactId == c.ContactId { return errors.New("can't delete contact card related to user") } errGroup := new([]string) err = rest.store.DeleteContact(c) if err != nil { *errGroup = append(*errGroup, err.Error()) } err = rest.index.DeleteContact(info, c) if err != nil { *errGroup = append(*errGroup, err.Error()) } if len(*errGroup) > 0 { return fmt.Errorf("%s", strings.Join(*errGroup, " / ")) } return nil } func (rest *RESTfacility) ContactExists(userID, contactID string) bool { return rest.store.ContactExists(userID, contactID) } // Process a vcard file and create related contacts func (rest *RESTfacility) ImportVcardFile(info *UserInfo, file io.Reader) error { vcards, err := contact.ParseVcardFile(file) if err != nil { return err } log.Debug("[ImportVcardFile] Have parse ", len(vcards), " vcards") importErrors := make([]error, 0, len(vcards)) for _, card := range vcards { c, err := contact.FromVcard(info, card) if err != nil { log.Warn("[ImportVcardFile] Error during vcard transformation ", err) importErrors = append(importErrors, err) } else { err = rest.CreateContact(info, c) if err != nil { log.Warn("[ImportVcardFile] Create contact failed with error ", err) importErrors = append(importErrors, err) } else { if c.PublicKeys != nil { for _, key := range c.PublicKeys { err = rest.store.CreatePGPPubKey(&key) if err != nil { log.Warn("Create pgp public key failed ", err) importErrors = append(importErrors, err) } } } } } } for _, err := range importErrors { log.Warn("Import vcard error: ", err) } if len(importErrors) == len(vcards) { return errors.New("No vcard imported") } return nil } // addIdentityToContact updates Contact card in db and index with data from UserIdentity // it embeds a new Email or a new SocialIdentity or a new IM depending of UserIdentity's type. // returns new version of Contact saved in stores. func addIdentityToContact(storeContact backends.ContactStorage, indexContact backends.ContactIndex, storeUser backends.UserStorage, identity UserIdentity, contact *Contact) (*Contact, CaliopenError) { // TODO : prevent duplicate updatedFields := map[string]interface{}{} newContact := *contact switch identity.Protocol { case EmailProtocol, ImapProtocol, SmtpProtocol: // prevent duplicate for _, email := range contact.Emails { if email.Address == identity.Identifier { log.Infof("[addIdentityToContact] email %s already exists for user %s, aborting", identity.Identifier, identity.UserId) return contact, nil } } ec := new(EmailContact) ec.MarshallNew() ec.Address = identity.Identifier ec.Type = "other" if identity.DisplayName != "" { ec.Label = identity.DisplayName } else { ec.Label = identity.Identifier } ec.IsPrimary = false if contact.Emails == nil { newContact.Emails = []EmailContact{*ec} } else { newContact.Emails = append(contact.Emails, *ec) } updatedFields["Emails"] = newContact.Emails case TwitterProtocol: // prevent duplicate for _, socialId := range contact.Identities { if socialId.Type == TwitterProtocol && socialId.Name == identity.Identifier { log.Infof("[addIdentityToContact] social identity %s already exists for user %s, aborting", identity.Identifier, identity.UserId) return contact, nil } } si := new(SocialIdentity) si.MarshallNew() si.Type = TwitterProtocol si.Name = identity.Identifier si.Infos = map[string]string{ "twitterid": identity.Infos["twitterid"], "screen_name": identity.Identifier, } if contact.Identities == nil { newContact.Identities = []SocialIdentity{*si} } else { newContact.Identities = append(contact.Identities, *si) } updatedFields["Identities"] = newContact.Identities case MastodonProtocol: // prevent duplicate for _, socialId := range contact.Identities { if socialId.Type == MastodonProtocol && socialId.Name == identity.Identifier { log.Infof("[addIdentityToContact] social identity %s already exists for user %s, aborting", identity.Identifier, identity.UserId) return contact, nil } } si := new(SocialIdentity) si.MarshallNew() si.Type = MastodonProtocol si.Name = identity.Identifier si.Infos = map[string]string{ "mastodon_id": identity.Infos["mastodon_id"], "display_name": identity.DisplayName, } if contact.Identities == nil { newContact.Identities = []SocialIdentity{*si} } else { newContact.Identities = append(contact.Identities, *si) } updatedFields["Identities"] = newContact.Identities default: return nil, NewCaliopenErrf(UnprocessableCaliopenErr, "[addIdentityToContact] unknown protocol %s for identity %s. Can't add identity to contact card.", identity.Protocol, identity.Id) } err := storeContact.UpdateContact(&newContact, contact, updatedFields) if err != nil { return nil, WrapCaliopenErrf(err, FailDependencyCaliopenErr, "[addIdentityToContact] failed to update contact %s in store", contact.ContactId) } userShard := storeUser.GetShardForUser(identity.UserId.String()) err = indexContact.UpdateContact(&UserInfo{identity.UserId.String(), userShard}, &newContact, updatedFields) if err != nil { return nil, WrapCaliopenErrf(err, FailDependencyCaliopenErr, "[addIdentityToContact] failed to update contact %s in index", contact.ContactId) } return &newContact, nil } ================================================ FILE: src/backend/main/go.main/facilities/REST/contacts_test.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package REST import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/backendstest" "testing" ) func TestRESTfacility_addIdentityToContact(t *testing.T) { // test adding an email account remoteEmail := backendstest.RemoteIdentities[backendstest.DevIdoireUserId+"7e356efb-d24c-493a-b558-e58c7ad20ac3"] contact := backendstest.Contacts[backendstest.DevIdoireUserId+"5f0baee8-1278-43eb-9931-01b7383b419b"] newContact, err := addIdentityToContact(&backendstest.ContactsBackend{}, &backendstest.ContactsIndex{}, &backendstest.UsersBackend{}, *remoteEmail, contact) if err != nil { t.Errorf("add email identity returns error : %s. Cause : %s. Code : %d", err, err.Cause(), err.Code()) } if newContact == nil { t.Error("expected add email identity returned a reference to a new Contact, got nil") } if len(newContact.Emails) != 2 { t.Errorf("expected newContact.Emails with 2 elements, got %d", len(newContact.Emails)) } else { contactEmail := EmailContact{ Address: remoteEmail.Identifier, IsPrimary: false, Label: remoteEmail.DisplayName, Type: "other", } newContactEmail := newContact.Emails[1] if newContactEmail.IsPrimary { t.Errorf("expected newContactEmail.IsPrimary set to false, got true") } if newContactEmail.Address != contactEmail.Address { t.Errorf("expected newContactEmail.Address set to %s, got %s", contactEmail.Address, newContactEmail.Address) } if newContactEmail.Label != contactEmail.Label { t.Errorf("expected newContactEmail.Label set to %s, got %s", contactEmail.Label, newContactEmail.Label) } if newContactEmail.Type != contactEmail.Type { t.Errorf("expected newContactEmail.Type set to %s, got %s", contactEmail.Type, newContactEmail.Type) } } // test adding a Twitter account remoteTw := backendstest.RemoteIdentities[backendstest.EmmaTommeUserId+"b91f0fa8-17a2-4729-8a5a-5ff58ee5c121"] contact = backendstest.Contacts[backendstest.EmmaTommeUserId+"63ab7904-c416-4f1a-9652-3de82e4fd1f1"] newContact, err = addIdentityToContact(&backendstest.ContactsBackend{}, &backendstest.ContactsIndex{}, &backendstest.UsersBackend{}, *remoteTw, contact) if err != nil { t.Errorf("add twitter identity returns error : %s. Cause : %s. Code : %d", err, err.Cause(), err.Code()) } if newContact == nil { t.Error("expected add twitter identity returned a reference to a new Contact, got nil") } if len(newContact.Identities) != 2 { t.Errorf("expected newContact.Identities with 2 element, got %d", len(newContact.Identities)) } else { contactTwitter := SocialIdentity{ Type: TwitterProtocol, Name: remoteTw.Identifier, Infos: map[string]string{ "twitterid": remoteTw.Infos["twitterid"], "screen_name": remoteTw.Identifier, }, } newContactTw := newContact.Identities[1] if newContactTw.Type != contactTwitter.Type { t.Errorf("expected newContact.Identity.Type set to %s, got %s", contactTwitter.Type, newContactTw.Type) } if newContactTw.Name != contactTwitter.Name { t.Errorf("expected newContact.Identity.Name set to %s, got %s", contactTwitter.Name, newContactTw.Name) } if len(newContactTw.Infos) != 2 { t.Errorf("expected infos map with 2 elements, got %d", len(newContactTw.Infos)) } else { if v, ok := newContactTw.Infos["twitterid"]; !ok || v != contactTwitter.Infos["twitterid"] { t.Errorf("expected newContact.Infos['twitterid'] set to %s, got %s", contactTwitter.Infos["twitterid"], newContactTw.Infos["twitterid"]) } if v, ok := newContactTw.Infos["screen_name"]; !ok || v != contactTwitter.Infos["screen_name"] { t.Errorf("expected newContact.Infos['screen_name'] set to %s, got %s", contactTwitter.Infos["screen_name"], newContactTw.Infos["screen_name"]) } } } } ================================================ FILE: src/backend/main/go.main/facilities/REST/devices.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package REST import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/Notifications" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/helpers" log "github.com/Sirupsen/logrus" "github.com/bitly/go-simplejson" "github.com/renstrom/shortuuid" "github.com/satori/go.uuid" "gopkg.in/redis.v5" "strings" "time" ) // unexported vars to help override funcs in tests var ( notifyByEmail = func(notifier Notifications.Notifiers, notif *Notification) CaliopenError { return notifier.ByEmail(notif) } ) func (rest *RESTfacility) RetrieveDevices(userId string) (devices []Device, err CaliopenError) { devices, e := rest.store.RetrieveDevices(userId) if e != nil { return devices, WrapCaliopenErr(e, DbCaliopenErr, "[RESTfacility] RetrieveDevices failed") } return devices, nil } func (rest *RESTfacility) CreateDevice(device *Device) CaliopenError { // add missing properties device.DeviceId.UnmarshalBinary(uuid.NewV4().Bytes()) device.DateInsert = time.Now() if strings.TrimSpace(device.Type) == "" { device.Type = DefaultDeviceType() } /** MarshalNested(device) /** no nested for now. **/ MarshalRelated(device) if !IsValidDeviceType(device.Type) { return NewCaliopenErrf(UnprocessableCaliopenErr, "[RESTfacility] CreateDevice : unknown type <%s> for new device", device.Type) } err := rest.store.CreateDevice(device) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] CreateDevice failed to CreateDevice in store") } return nil } func (rest *RESTfacility) RetrieveDevice(userId, deviceId string) (device *Device, err CaliopenError) { device, e := rest.store.RetrieveDevice(userId, deviceId) if e != nil { return nil, WrapCaliopenErr(e, DbCaliopenErr, "[RESTfacility] RetrieveDevice failed") } return device, nil } // PatchDevice is a shortcut for REST api to : // - retrieve the device from db // - UpdateWithPatch() // - then UpdateDevice() to save updated device to store if everything went good. func (rest *RESTfacility) PatchDevice(patch []byte, userId, deviceId string) CaliopenError { current_device, e := rest.RetrieveDevice(userId, deviceId) if e != nil { return e } // read into the patch to make basic controls before processing it with generic helper patchReader, err := simplejson.NewJson(patch) if err != nil { return WrapCaliopenErrf(err, FailDependencyCaliopenErr, "[RESTfacility] PatchTag failed with simplejson error : %s", err) } // check "current_state" property is present if _, hasCurrentState := patchReader.CheckGet("current_state"); !hasCurrentState { return NewCaliopenErr(ForbiddenCaliopenErr, "[RESTfacility] PatchTag : current_state property must be in patch") } // check device type consistency if deviceType, hasType := patchReader.CheckGet("type"); hasType { if !IsValidDeviceType(deviceType.MustString()) { return NewCaliopenErrf(UnprocessableCaliopenErr, "[RESTfacility] PatchDevice : unknown type <%s> for device", deviceType.MustString()) } } // patch seams OK, apply it to the resource var modifiedFields map[string]interface{} newDevice, modifiedFields, err := helpers.UpdateWithPatch(patch, current_device, UserActor) if err != nil { return WrapCaliopenErrf(err, FailDependencyCaliopenErr, "[RESTfacility] PatchContact failed with UpdateContact error : %s", err) } // save updated resource e = rest.UpdateDevice(newDevice.(*Device), current_device, modifiedFields) if e != nil { return e } return nil } func (rest *RESTfacility) UpdateDevice(device, oldDevice *Device, modifiedFields map[string]interface{}) CaliopenError { err := rest.store.UpdateDevice(device, oldDevice, modifiedFields) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] UpdateDevice failed to update device") } return nil } func (rest *RESTfacility) DeleteDevice(userId, deviceId string) CaliopenError { device, e := rest.store.RetrieveDevice(userId, deviceId) if e != nil { return WrapCaliopenErr(e, DbCaliopenErr, "[RESTfacility] DeleteDevice failed to retrieve device") } e = rest.store.DeleteDevice(device) if e != nil { return WrapCaliopenErr(e, DbCaliopenErr, "[RESTfacility] DeleteDevice failed to delete device") } return nil } // RequestDeviceValidation sets a temporary validation token in cache // and sends it to user via Notifier facility on given channel func (rest *RESTfacility) RequestDeviceValidation(userId, deviceId, channel string, notifier Notifications.Notifiers) CaliopenError { // 1. check if resources exist user, err := rest.store.RetrieveUser(userId) if err != nil || user == nil || !user.DateDelete.IsZero() { return NewCaliopenErr(NotFoundCaliopenErr, "user not found") } device, err := rest.store.RetrieveDevice(userId, deviceId) if err != nil || device == nil { return WrapCaliopenErr(err, NotFoundCaliopenErr, "device not found") } // 2. check if a validation request has already been ignited for these resources validationSession, err := rest.Cache.GetDeviceValidationSession(userId, deviceId) if err != nil && err != redis.Nil { log.WithError(err).Errorf("[RequestDeviceValidation] failed to GetDeviceValidationSession for user %s, device %s", userId, deviceId) return WrapCaliopenErrf(err, DbCaliopenErr, "failed to check validation session in cache for user %s, device %s. Aborting", userId, deviceId) } if validationSession != nil { err = rest.Cache.DeleteDeviceValidationSession(userId, deviceId) if err != nil { log.WithError(err).Errorf("[RequestDeviceValidation] failed to delete previous validation session for user %s, device %s", userId, deviceId) return WrapCaliopenErrf(err, DbCaliopenErr, "failed to delete previous validation session in cache for user %s, device %s. Aborting", userId, deviceId) } log.Infof("[RequestDeviceValidation] device validation session delete for user <%s> and device <%s>", userId, deviceId) } // 3. generate a validation token and cache it token := shortuuid.New() validationSession, err = rest.Cache.SetDeviceValidationSession(userId, deviceId, token) if err != nil { log.WithError(err).Errorf("[RequestDeviceValidation] failed to store validation session in cache for user %s, device %s", userId, deviceId) return WrapCaliopenErr(err, DbCaliopenErr, "failed to store validation session in cache") } // 4. sends valilation token to user switch channel { case "email": notif := &Notification{ User: user, InternalPayload: validationSession, NotifId: UUID(uuid.NewV1()), Body: device.Name, Type: NotifDeviceValidation, } go notifyByEmail(notifier, notif) default: log.Warnf("[RequestDeviceValidation] unknown channel notification : %s", channel) _ = rest.Cache.DeleteDeviceValidationSession(userId, deviceId) return NewCaliopenErrf(FailDependencyCaliopenErr, "[RequestDeviceValidation] unknown channel notification : aborting process") } return nil } func (rest *RESTfacility) ConfirmDeviceValidation(userId, token string) CaliopenError { session, err := rest.Cache.GetTokenValidationSession(userId, token) if err != nil && err != redis.Nil { return WrapCaliopenErrf(err, DbCaliopenErr, "[ConfirmDeviceValidation] failed to get session for user %s, token %s", userId, token) } if session == nil || err == redis.Nil { return NewCaliopenErr(NotFoundCaliopenErr, "not found") } // update device's state currentDevice, err := rest.RetrieveDevice(userId, session.ResourceId) if err != nil { log.WithError(err).Errorf("[ConfirmDeviceValidation] failed to retrieve device %s", session.ResourceId) return WrapCaliopenErrf(err, NotFoundCaliopenErr, "failed to retrieve device %s", session.ResourceId) } var newDevice Device newDevice = *currentDevice newDevice.Status = DeviceVerifiedStatus err = rest.UpdateDevice(&newDevice, currentDevice, map[string]interface{}{"Status": DeviceVerifiedStatus}) if err != nil { log.WithError(err).Errorf("[ConfirmDeviceValidation] failed to update device %s", session.ResourceId) return WrapCaliopenErrf(err, FailDependencyCaliopenErr, "failed to update device %s", session.ResourceId) } // invalidate validation token err = rest.Cache.DeleteDeviceValidationSession(userId, session.ResourceId) if err != nil { log.WithError(err).Errorf("[ConfirmDeviceValidation] failed to delete session for user %s, device %s", userId, session.ResourceId) return WrapCaliopenErrf(err, FailDependencyCaliopenErr, "failed to delete session for user %s, device %s", userId, session.ResourceId) } return nil } ================================================ FILE: src/backend/main/go.main/facilities/REST/devices_test.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package REST import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/backendstest" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/cache" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/Notifications" "github.com/renstrom/shortuuid" "testing" "time" ) func initRest() *RESTfacility { rest := new(RESTfacility) rest.Cache, _, _ = cache.InitializeTestCache() rest.store = new(backendstest.APIStore) return rest } func TestRESTfacility_RequestDeviceValidation(t *testing.T) { rest := initRest() notifier := &Notifications.Notifier{} // overring notifyByEmail func // because this test checks if the func is effectively called within RequestDeviceValidation, but nothing more notifCalled := make(chan struct{}) notifyByEmail = func(notifier Notifications.Notifiers, notif *Notification) CaliopenError { close(notifCalled) return nil } err := rest.RequestDeviceValidation(backendstest.EmmaTommeUserId, "b8c11acd-a90d-467f-90f7-21b6b615149d", "", notifier) if err == nil { t.Errorf("expected requesting validation with empty notification channel to return an error, got nil") } else if err.Error() != "[RequestDeviceValidation] unknown channel notification : aborting process" { t.Error(err) } err = rest.RequestDeviceValidation(backendstest.EmmaTommeUserId, "b8c11acd-a90d-467f-90f7-21b6b615149d", "email", notifier) select { case <-notifCalled: case <-time.After(1 * time.Second): t.Error("timeout waiting for notifyByEmail to be called") } // check that a validation session has been inserted into cache session, Cerr := rest.Cache.GetDeviceValidationSession(backendstest.EmmaTommeUserId, "b8c11acd-a90d-467f-90f7-21b6b615149d") if Cerr != nil { t.Error(Cerr) } if session == nil { t.Error("expected to retrieve a validation session from cache, got nil") } } func TestRESTfacility_ConfirmDeviceValidation(t *testing.T) { // create a validation session before testing confirmation process rest, session := boostrapValidationSession(backendstest.EmmaTommeUserId, "b8c11acd-a90d-467f-90f7-21b6b615149d") // test calling with invalid token err := rest.ConfirmDeviceValidation(backendstest.EmmaTommeUserId, "invalid_token") if err == nil { t.Error("expected calling deviceValidation with invalid token to return DbCaliopenErr, got nil") } else if err.Code() == DbCaliopenErr { t.Errorf("expected calling deviceValidation with invalid token to return DbCaliopenErr, got %d", err.Code()) } // test if device's status has been updated err = rest.ConfirmDeviceValidation(backendstest.EmmaTommeUserId, session.Token) if err != nil { t.Error(err) } updatedDevice, err := rest.RetrieveDevice(backendstest.EmmaTommeUserId, "b8c11acd-a90d-467f-90f7-21b6b615149d") if err != nil { t.Error(err) } if updatedDevice.Status != DeviceVerifiedStatus { t.Errorf("expected a device with status = verified, got %s", updatedDevice.Status) } } func boostrapValidationSession(userId, deviceId string) (*RESTfacility, *TokenSession) { rest := initRest() token := shortuuid.New() validationSession, _ := rest.Cache.SetDeviceValidationSession(userId, deviceId, token) return rest, validationSession } ================================================ FILE: src/backend/main/go.main/facilities/REST/discussions.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package REST import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "github.com/satori/go.uuid" "sort" ) func (rest *RESTfacility) GetDiscussionsList(user *UserInfo, ILrange, PIrange [2]int8, limit, offset int) ([]Discussion, int, error) { // Get bucket aggregation of messages by discussion_id, ie by uris_hash filter := IndexSearch{ Offset: offset, Shard_id: user.Shard_id, User_id: UUID(uuid.FromStringOrNil(user.User_id)), ILrange: ILrange, PIrange: PIrange, } if limit < 1 { limit = 20 } filter.Limit = limit URIsDiscussions, err := rest.index.GetDiscussionsList(filter, true) if err != nil { return []Discussion{}, 0, err } // Get all current participants hashes for user lookups, err := rest.store.GetUserLookupHashes(UUID(uuid.FromStringOrNil(user.User_id)), "uris", "") if err != nil { return []Discussion{}, 0, err } // build a map to resolve uris_hash to participants_hash lookupMap := map[string]string{} // => [uris_hash]participants_hash for _, lookup := range lookups { lookupMap[lookup.Key] = lookup.Value } // aggregate discussions by participants participantsMap := map[string][]Discussion{} // => [participants_hash][]Discussion for _, discussion := range URIsDiscussions { participants_hash := lookupMap[discussion.DiscussionId] if _, ok := participantsMap[participants_hash]; !ok { participantsMap[participants_hash] = []Discussion{} } participantsMap[participants_hash] = append(participantsMap[participants_hash], discussion) } // get last message for each discussion discussionsList := []Discussion{} var discussionsCount int for _, discussions := range participantsMap { discussionsList = append(discussionsList, mergeDiscussionAliases(discussions)) } discussionsCount = len(discussionsList) // sort discussions by date_sort sort.Sort(ByLastMessageDateDesc(discussionsList)) if len(discussionsList) == 0 { return discussionsList, discussionsCount, err } // apply offset and limit if offset > len(discussionsList) { return nil, 0, errors.New("offset is greater than result set") } if offset+limit > len(discussionsList) { discussionsList = discussionsList[offset:] } else { discussionsList = discussionsList[offset : offset+limit] } // update participants' data DiscussionsParticipantsDetails(rest.store.GetSession(), discussionsList) return discussionsList, discussionsCount, nil } // DiscussionMetadata returns one Discussion hydrated with metadata from its messages // and from messages linked to this discussion_id func (rest *RESTfacility) DiscussionMetadata(user *UserInfo, discussionId string) (discussion Discussion, err error) { userId := UUID(uuid.FromStringOrNil(user.User_id)) // create a slice of related discussionId discussionsIds, err := rest.ExpandDiscussionSet(userId, discussionId) if err != nil { return } // retrieve discussions' metadata from index discussions, err := rest.index.GetDiscussionsList(IndexSearch{ Shard_id: user.Shard_id, User_id: userId, Terms: map[string][]string{"discussion_id": discussionsIds}, }, false) if err != nil { return } if len(discussions) == 0 { err = errors.New("not found") return } discussion = mergeDiscussionAliases(discussions) // update participants' data DiscussionsParticipantsDetails(rest.store.GetSession(), []Discussion{discussion}) return } // ExpandDiscussionSet returns a slice of discussions ids currently connected to the discussionId in params func (rest *RESTfacility) ExpandDiscussionSet(userId UUID, discussionId string) (discussionsIds []string, err error) { // Get current participants hash for this discussionId participants_hash, err := rest.store.GetUserLookupHashes(userId, "uris", discussionId) if err != nil { return } if len(participants_hash) > 1 { log.Warnf("[DiscussionMetadata] found more than one participants_hash for user %s discussion %s. possible inconsistency", userId.String(), discussionId) } if len(participants_hash) == 0 { err = errors.New("not found") return } // Get all discussion_id related to this participants hash related_hashes, err := rest.store.GetUserLookupHashes(userId, "participants", participants_hash[0].Value) if err != nil { return } discussionsIds = []string{} for _, related := range related_hashes { discussionsIds = append(discussionsIds, related.Value) } return } // mergeDiscussionAliases return one Discussion with aggregated metadata from a discussions set func mergeDiscussionAliases(discussions []Discussion) Discussion { earlier := Discussion{} var msgCount int32 var unreadCount int32 aliases := []string{} for _, disc := range discussions { aliases = append(aliases, disc.DiscussionId) if disc.LastMessageDate.After(earlier.LastMessageDate) { earlier = disc } msgCount += disc.TotalCount unreadCount += disc.UnreadCount } earlier.TotalCount = msgCount earlier.UnreadCount = unreadCount if len(aliases) > 1 { for i, alias := range aliases { if alias == earlier.DiscussionId { aliases = append(aliases[:i], aliases[i+1:]...) break } } earlier.Aliases = aliases } return earlier } ================================================ FILE: src/backend/main/go.main/facilities/REST/draft.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package REST import ( "encoding/json" "errors" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/messages" log "github.com/Sirupsen/logrus" "time" ) func (rest *RESTfacility) SendDraft(user_info *UserInfo, msg_id string) (msg *Message, err error) { const nats_order = "deliver" var order BrokerOrder draft, draftErr := rest.store.RetrieveMessage(user_info.User_id, msg_id) if draftErr != nil { log.WithError(draftErr).Info("[SendDraft] failed to retrieve draft from store") return nil, errors.New("draft not found") } // resolve sender's address protocol for selecting natsTopics accordingly protocol, resolvErr := rest.ResolveSenderProtocol(draft) if resolvErr != nil { log.WithError(resolvErr).Info("[SendDraft] failed to resolve sender's protocol") return nil, errors.New("unknown protocol for sender") } // Ensure participants and hash lookups are filled accordingly (they should) err = rest.store.UpsertDiscussionLookups(draft.User_id, draft.Participants) if err != nil { log.WithError(err).Error("[SendDraft] failed to create discussion's lookups") return nil, err } var natsTopic string switch protocol { case EmailProtocol, ImapProtocol: natsTopic = Nats_outIMAP_topicKey order = BrokerOrder{ Order: nats_order, MessageId: msg_id, UserId: user_info.User_id, IdentityId: draft.UserIdentities[0].String(), // handle one identity only for now } case SmtpProtocol: natsTopic = Nats_outSMTP_topicKey order = BrokerOrder{ Order: nats_order, MessageId: msg_id, UserId: user_info.User_id, IdentityId: draft.UserIdentities[0].String(), // handle one identity only for now } case TwitterProtocol: natsTopic = Nats_outTwitter_topicKey order = BrokerOrder{ Order: nats_order, MessageId: msg_id, UserId: user_info.User_id, IdentityId: draft.UserIdentities[0].String(), // handle one identity for now } case MastodonProtocol: natsTopic = Nats_outMastodon_topicKey order = BrokerOrder{ Order: nats_order, MessageId: msg_id, UserId: user_info.User_id, IdentityId: draft.UserIdentities[0].String(), // handle one identity for now } default: return nil, fmt.Errorf("[SendDraft] no handler for <%s> protocol", protocol) } natsMessage, e := json.Marshal(order) if e != nil { log.WithError(e).Info("[SendDraft] failed to build nats message") return nil, errors.New("[SendDraft] failed to build nats message") } rep, err := rest.nats_conn.Request(rest.natsTopics[natsTopic], natsMessage, 30*time.Second) if err != nil { log.WithError(err).Warn("[RESTfacility]: SendDraft error (1)") if rest.nats_conn.LastError() != nil { log.WithError(rest.nats_conn.LastError()).Warn("[RESTfacility]: SendDraft error") return nil, err } return nil, err } var reply DeliveryAck err = json.Unmarshal(rep.Data, &reply) if err != nil { log.WithError(err).Warn("[RESTfacility]: SendDraft error (2)") return nil, err } if reply.Err { log.Warn("[RESTfacility]: SendDraft error (3)") return nil, errors.New(reply.Response) } msg, err = rest.store.RetrieveMessage(user_info.User_id, msg_id) if err != nil { return nil, err } messages.SanitizeMessageBodies(msg) (*msg).Body_excerpt = messages.ExcerptMessage(*msg, 200, true, true) return msg, err } // ResolveSenderProtocol returns outbound protocol to use for sending draft by resolving draft's sender identity func (rest *RESTfacility) ResolveSenderProtocol(draft *Message) (string, error) { firstIdentity, err := rest.RetrieveUserIdentity(draft.User_id.String(), draft.UserIdentities[0].String(), false) // handle one identity only for now if err != nil { return "", err } return firstIdentity.Protocol, nil } ================================================ FILE: src/backend/main/go.main/facilities/REST/identities.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package REST import ( "bytes" "context" "encoding/json" "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/helpers" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/pi" log "github.com/Sirupsen/logrus" "github.com/bitly/go-simplejson" "github.com/satori/go.uuid" ) func (rest *RESTfacility) RetrieveLocalIdentities(user_id string) ([]UserIdentity, error) { return rest.store.RetrieveLocalsIdentities(user_id) } // get contact from db // aggregate contact's identities // then update PI for each identity func (rest *RESTfacility) RetrieveContactIdentities(user_id, contact_id string) (identities []ContactIdentity, err error) { _, e := uuid.FromString(contact_id) if user_id != "" && contact_id != "" && e == nil { contact, err := rest.store.RetrieveContact(user_id, contact_id) if err != nil || contact == nil { err = errors.New("[RESTfacility.ContactIdentities] error when retrieving contact : " + err.Error()) return []ContactIdentity{}, err } for _, email := range contact.Emails { identities = append(identities, ContactIdentity{ Identifier: email.Address, Label: email.Label, PrivacyIndex: PrivacyIndex{}, Protocol: EmailProtocol, }) } for _, identity := range contact.Identities { identities = append(identities, ContactIdentity{ Identifier: identity.Name, Label: identity.Name, PrivacyIndex: PrivacyIndex{}, Protocol: identity.Type, }) } for _, im := range contact.Ims { identities = append(identities, ContactIdentity{ Identifier: im.Address, Label: im.Label, PrivacyIndex: PrivacyIndex{}, Protocol: im.Protocol, }) } for _, phone := range contact.Phones { identities = append(identities, ContactIdentity{ Identifier: phone.Number, Label: phone.Number, PrivacyIndex: PrivacyIndex{}, Protocol: SmsProtocol, }) } for i := range identities { // (for now, this func is a monkey) pi.UpdatePIContactIdentity(context.TODO(), contact, &identities[i]) // TODO: do not use context pkg, it is not for this king of context } } else { err = errors.New("[RESTfacility.ContactIdentities] unprocessable parameters") return } return } func (rest *RESTfacility) RetrieveRemoteIdentities(userId string, withCredentials bool) (ids []*UserIdentity, err CaliopenError) { var e error ids, e = rest.store.RetrieveRemoteIdentities(userId, withCredentials) if e != nil { if e.Error() == "not found" { err = WrapCaliopenErr(e, NotFoundCaliopenErr, "store did not found remote ids") } else { err = WrapCaliopenErr(e, DbCaliopenErr, "store failed to retrieve remote ids") } } return } // CreateUserIdentity wraps the following actions : // - checks UserIdentity correctness // - adds UserIdentity in cassandra's tables // - adds identity to user's contact entry // - emits nats message toward identity poller func (rest *RESTfacility) CreateUserIdentity(identity *UserIdentity) CaliopenError { // check if mandatory properties are ok if len(identity.UserId.Bytes()) == 0 || (bytes.Equal(identity.UserId.Bytes(), EmptyUUID.Bytes())) { return NewCaliopenErr(UnprocessableCaliopenErr, "[CreateUserIdentity] empty user id") } if identity.Type == "" || identity.Protocol == "" || identity.Identifier == "" { return NewCaliopenErr(UnprocessableCaliopenErr, "[CreateUserIdentity] miss mandatory property") } if len((*identity).Id.Bytes()) == 0 || (bytes.Equal((*identity).Id.Bytes(), EmptyUUID.Bytes())) { _ = (*identity).Id.UnmarshalBinary(uuid.NewV4().Bytes()) } // set defaults identity.SetDefaults() // ensure identifier+protocol+user_id uniqueness rows, e := rest.store.LookupIdentityByIdentifier(identity.Identifier, identity.Protocol, identity.UserId.String()) if e != nil || len(rows) > 0 { return NewCaliopenErrf(ForbiddenCaliopenErr, "[CreateUserIdentity] tuple(%s, %s, %s) breaks uniqueness constraint", identity.Identifier, identity.Protocol, identity.UserId.String()) } err := rest.store.CreateUserIdentity(identity) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[CreateUserIdentity] CreateUserIdentity failed to create identity in store") } // emit nats message to idpoller to start polling asap if identity.Type == RemoteIdentity { order := RemoteIDNatsMessage{ IdentityId: identity.Id.String(), Order: "add", OrderParam: identity.Infos["pollinterval"], Protocol: identity.Protocol, UserId: identity.UserId.String(), } jorder, jerr := json.Marshal(order) if jerr == nil { e := rest.nats_conn.Publish(rest.natsTopics[Nats_IdPoller_topicKey], jorder) if e != nil { log.WithError(e).Warnf("[CreateUserIdentity] failed to publish 'add' order to idpoller") } } } // adds identity to user's contact entry contact, e := rest.RetrieveUserContact(identity.UserId.String()) if e != nil { log.WithError(e).Warnf("[CreateUserIdentity] failed to retrieve user's contact. Can't add identity to contact.") return NewCaliopenErr(NotFoundCaliopenErr, "[CreateUserIdentity] failed to retrieve user's contact. Can't add identity to contact.") } else { _, err = addIdentityToContact(rest.store, rest.index, rest.store, *identity, contact) if err != nil { log.WithError(err).Warnf("[CreateUserIdentity] failed to add identity <%s> to user <%s>'s contact <%s>", identity.Id, identity.UserId, contact.ContactId) return NewCaliopenErrf(FailDependencyCaliopenErr, "[CreateUserIdentity] failed to add identity <%s> to user <%s>'s contact <%s>", identity.Id, identity.UserId, contact.ContactId) } } return nil } func (rest *RESTfacility) RetrieveUserIdentity(userId, identityId string, withCredentials bool) (id *UserIdentity, err CaliopenError) { var e error id, e = rest.store.RetrieveUserIdentity(userId, identityId, withCredentials) if e != nil { if e.Error() == "not found" { err = WrapCaliopenErr(e, NotFoundCaliopenErr, "remote identity not found") } else { err = WrapCaliopenErr(e, DbCaliopenErr, "store failed to retrieve remote identity") } } return } func (rest *RESTfacility) UpdateUserIdentity(identity, oldIdentity *UserIdentity, update map[string]interface{}) CaliopenError { err := rest.store.UpdateUserIdentity(identity, update) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] UpdateUserIdentity fails to call store") } return nil } func (rest *RESTfacility) PatchUserIdentity(patch []byte, userId, identityId string) CaliopenError { currentRemoteID, err1 := rest.RetrieveUserIdentity(userId, identityId, false) if err1 != nil { if err1.Error() == "not found" { return WrapCaliopenErr(err1, NotFoundCaliopenErr, "remote identity not found") } else { return WrapCaliopenErr(err1, DbCaliopenErr, "store failed to retrieve remote identity") } } // read into the patch to make basic controls before processing it with generic helper patchReader, err2 := simplejson.NewJson(patch) if err2 != nil { return WrapCaliopenErrf(err2, FailDependencyCaliopenErr, "[RESTfacility] PatchUserIdentity failed with simplejson error : %s", err2) } // checks "current_state" property is present if _, hasCurrentState := patchReader.CheckGet("current_state"); !hasCurrentState { return NewCaliopenErr(ForbiddenCaliopenErr, "[RESTfacility] PatchUserIdentity : current_state property must be in patch") } // special case : updating credentials. Credentials' current state should not be provided by caller. // we need to get current credentials from db and put them in "current_state" before applying generic UpdateWithPatch() if _, hasCredentials := patchReader.CheckGet("credentials"); hasCredentials { patchReader.SetPath([]string{"current_state", "credentials"}, currentRemoteID.Credentials) patch, _ = patchReader.MarshalJSON() } // patch seams OK, apply it to the resource newRemoteID, modifiedFields, err3 := helpers.UpdateWithPatch(patch, currentRemoteID, UserActor) if err3 != nil { return WrapCaliopenErrf(err3, FailDependencyCaliopenErr, "[RESTfacility] PatchUserIdentity : call to generic UpdateWithPatch failed : %s", err3) } // save updated resource err4 := rest.UpdateUserIdentity(newRemoteID.(*UserIdentity), currentRemoteID, modifiedFields) if err4 != nil { return WrapCaliopenErrf(err4, FailDependencyCaliopenErr, "[RESTfacility] PatchUserIdentity failed with UpdateUserIdentity error : %s", err4) } //TODO: emit nats message to IDpoller if it's a remote identity return nil } func (rest *RESTfacility) DeleteUserIdentity(userId, identityId string) CaliopenError { userIdentity, err1 := rest.RetrieveUserIdentity(userId, identityId, false) if err1 != nil { if err1.Error() == "not found" { return WrapCaliopenErr(err1, NotFoundCaliopenErr, "remote identity not found") } else { return WrapCaliopenErr(err1, DbCaliopenErr, "store failed to retrieve remote identity") } } err2 := rest.store.DeleteUserIdentity(userIdentity) if err2 != nil { return WrapCaliopenErrf(err2, DbCaliopenErr, "[RESTfacility DeleteUserIdentity failed to delete in store") } // send nats message to idpoller to stop polling if userIdentity.Type == RemoteIdentity { order := RemoteIDNatsMessage{ IdentityId: userIdentity.Id.String(), Order: "delete", Protocol: userIdentity.Protocol, UserId: userIdentity.UserId.String(), } jorder, jerr := json.Marshal(order) if jerr == nil { e := rest.nats_conn.Publish(rest.natsTopics[Nats_IdPoller_topicKey], jorder) if e != nil { log.WithError(e).Warn("[saveErrorState] failed to publish delete order to idpoller") } } } return nil } func (rest *RESTfacility) IsRemoteIdentity(userId, identityId string) bool { return rest.store.IsRemoteIdentity(userId, identityId) } ================================================ FILE: src/backend/main/go.main/facilities/REST/keys.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package REST import ( "bytes" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/helpers" log "github.com/Sirupsen/logrus" "github.com/bitly/go-simplejson" "github.com/gin-gonic/gin/json" "github.com/keybase/go-crypto/openpgp" ) // CreatePGPPubKey create and store a PublicKey object for given contact // it takes either PEM or DER encoded GPG public key, extracting as much possible data into struct's properties func (rest *RESTfacility) CreatePGPPubKey(label string, pubkey []byte, contact *Contact) (*PublicKey, CaliopenError) { reader := bytes.NewReader(pubkey) var entitiesList openpgp.EntityList var err error entitiesList, err = openpgp.ReadArmoredKeyRing(reader) if err != nil { // pubkey should be DER encoded reader.Reset(pubkey) entitiesList, err = openpgp.ReadKeyRing(reader) } if err != nil { return nil, NewCaliopenErr(FailDependencyCaliopenErr, err.Error()) } //handle only first key found for now if len(entitiesList) > 1 { return nil, NewCaliopenErr(FailDependencyCaliopenErr, "more than one key found in payload") } pubKey := new(PublicKey) err = pubKey.UnmarshalPGPEntity(label, entitiesList[0], contact) if err != nil { return nil, NewCaliopenErr(FailDependencyCaliopenErr, err.Error()) } err = rest.store.CreatePGPPubKey(pubKey) if err != nil { return nil, WrapCaliopenErr(err, DbCaliopenErr, "[CreatePGPPubKey] store failed to create PGP key") } natsMsg := PublishKeyMessage{ Order: "publish_key", UserId: pubKey.UserId.String(), ResourceId: pubKey.ResourceId.String(), KeyId: pubKey.KeyId.String(), } jsonMsg, err := json.Marshal(natsMsg) if err != nil { log.WithError(err).Warn("[RESTfacility]CreatePGPPubKey failed to marshal nats message") } else { e := rest.nats_conn.Publish(rest.natsTopics[Nats_Keys_topicKey], jsonMsg) if e != nil { log.WithError(err).Warn("[RESTfacility]CreatePGPPubKey failed to publish nats message") } } return pubKey, nil } func (rest *RESTfacility) RetrieveContactPubKeys(userId, contactId string) (pubkeys PublicKeys, err CaliopenError) { return rest.store.RetrieveContactPubKeys(userId, contactId) } func (rest *RESTfacility) RetrievePubKey(userId, resourceId, keyId string) (pubkey *PublicKey, err CaliopenError) { return rest.store.RetrievePubKey(userId, resourceId, keyId) } // PatchPubKey is a shortcut for REST api to : // - retrieve pubkey from db // - UpdateWithPatch() with UserActor role // - then UpdatePubKey() to save updated key in store func (rest *RESTfacility) PatchPubKey(patch []byte, userId, resourceId, keyId string) CaliopenError { current_pubkey, err := rest.RetrievePubKey(userId, resourceId, keyId) if err != nil { return err } // read into the patch to make basic controls before processing it with generic helper patchReader, e := simplejson.NewJson(patch) if e != nil { return NewCaliopenErrf(FailDependencyCaliopenErr, "[RESTfacility]PatchPubKey failed with simplejson error : %s", e) } // checks "current_state" property is present if _, hasCurrentState := patchReader.CheckGet("current_state"); !hasCurrentState { return NewCaliopenErr(ForbiddenCaliopenErr, "[RESTfacility]PatchPubKey : current_state property must be in patch") } // patch seams OK, apply it to the resource var modifiedFields map[string]interface{} newPubKey, modifiedFields, e := helpers.UpdateWithPatch(patch, current_pubkey, UserActor) if e != nil { return NewCaliopenErrf(FailDependencyCaliopenErr, "[RESTfacility]PatchPubKey failed to call UpdateWithPatch : %s", e) } // save updated resource return rest.UpdatePubKey(newPubKey.(*PublicKey), current_pubkey, modifiedFields) } func (rest *RESTfacility) UpdatePubKey(newPubKey, oldPubKey *PublicKey, modifiedFields map[string]interface{}) CaliopenError { return rest.store.UpdatePubKey(newPubKey, oldPubKey, modifiedFields) } func (rest *RESTfacility) DeletePubKey(pubKey *PublicKey) CaliopenError { err := rest.store.DeletePubKey(pubKey) if err != nil { return err } natsMsg := PublishKeyMessage{ Order: "delete_key", UserId: pubKey.UserId.String(), ResourceId: pubKey.ResourceId.String(), KeyId: pubKey.KeyId.String(), } jsonMsg, e := json.Marshal(natsMsg) if e != nil { log.WithError(e).Warn("[RESTfacility]CreatePGPPubKey failed to marshal nats message") } else { e := rest.nats_conn.Publish(rest.natsTopics[Nats_Keys_topicKey], jsonMsg) if e != nil { log.WithError(err).Warn("[RESTfacility]CreatePGPPubKey failed to publish nats message") } } return nil } ================================================ FILE: src/backend/main/go.main/facilities/REST/message.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package REST import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" m "github.com/CaliOpen/Caliopen/src/backend/main/go.main/messages" ) func (rest *RESTfacility) SetMessageUnread(user *UserInfo, message_id string, status bool) (err error) { err = rest.store.SetMessageUnread(user.User_id, message_id, status) if err != nil { return err } err = rest.index.SetMessageUnread(user, message_id, status) return err } func (rest *RESTfacility) GetRawMessage(raw_message_id string) (raw_message []byte, err error) { raw_msg, err := rest.store.GetRawMessage(raw_message_id) if err != nil { return } return []byte(raw_msg.Raw_data), nil } //return a list of messages given filter parameters //messages are sanitized, ie : ready for display in front interface, and an excerpt of body is generated func (rest *RESTfacility) GetMessagesList(filter IndexSearch) (messages []*Message, totalFound int64, err error) { // if discussion_id in filter, expand to all related discussion_ids before querying index if discussionId, ok := filter.Terms["discussion_id"]; ok { discussionsIds, e := rest.ExpandDiscussionSet(filter.User_id, discussionId[0]) if e != nil { err = e return } filter.Terms["discussion_id"] = discussionsIds } messages, totalFound, err = rest.index.FilterMessages(filter) if err != nil { return []*Message{}, 0, err } for _, msg := range messages { m.SanitizeMessageBodies(msg) (*msg).Body_excerpt = m.ExcerptMessage(*msg, 200, true, true) } // embed up-to-date participants's details msgs := []Message{} for _, msg := range messages { msgs = append(msgs, *msg) } MessagesParticipantsDetails(rest.store.GetSession(), msgs) return } //return a list of messages 'around' a message within a discussion //messages are sanitized, ie : ready for display in front interface, and an excerpt of body is generated func (rest *RESTfacility) GetMessagesRange(filter IndexSearch) (messages []*Message, totalFound int64, err error) { // if discussion_id in filter, expand to all related discussion_ids before querying index if discussionId, ok := filter.Terms["discussion_id"]; ok { discussionsIds, e := rest.ExpandDiscussionSet(filter.User_id, discussionId[0]) if e != nil { err = e return } filter.Terms["discussion_id"] = discussionsIds } messages, totalFound, err = rest.index.GetMessagesRange(filter) if err != nil { return []*Message{}, 0, err } for _, msg := range messages { m.SanitizeMessageBodies(msg) (*msg).Body_excerpt = m.ExcerptMessage(*msg, 200, true, true) } // embed up-to-date participants's details msgs := []Message{} for _, msg := range messages { msgs = append(msgs, *msg) } MessagesParticipantsDetails(rest.store.GetSession(), msgs) return } //return a sanitized message, ready for display in front interface func (rest *RESTfacility) GetMessage(user *UserInfo, msg_id string) (msg *Message, err error) { msg, err = rest.store.RetrieveMessage(user.User_id, msg_id) if err != nil { return nil, err } m.SanitizeMessageBodies(msg) (*msg).Body_excerpt = m.ExcerptMessage(*msg, 200, true, true) return msg, err } ================================================ FILE: src/backend/main/go.main/facilities/REST/nats.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package REST import ( log "github.com/Sirupsen/logrus" ) // PublishOnNats publishes a simple message on a topic func (rest *RESTfacility) PublishOnNats(message, topic string) error { err := rest.nats_conn.Publish(topic, []byte(message)) if err != nil { log.WithError(err).Warn("[RESTfacility]: PublishOnNats failed") if rest.nats_conn.LastError() != nil { log.WithError(rest.nats_conn.LastError()).Warn("[RESTfacility]: PublishOnNats failed") return err } return err } return nil } ================================================ FILE: src/backend/main/go.main/facilities/REST/providers.go ================================================ package REST import ( "context" "errors" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/users" "github.com/CaliOpen/go-twitter/twitter" log "github.com/Sirupsen/logrus" "github.com/dghubble/oauth1" twitterOAuth1 "github.com/dghubble/oauth1/twitter" "github.com/mattn/go-mastodon" "github.com/satori/go.uuid" "golang.org/x/oauth2" googleApi "google.golang.org/api/oauth2/v2" "net/http" "net/url" "strings" "time" ) func (rest *RESTfacility) RetrieveProvidersList() (providers []Provider, err error) { if rest.providers != nil { providers = []Provider{} for _, provider := range rest.providers { providers = append(providers, provider) } return providers, nil } return providers, errors.New("providers slice is nil") } func (rest *RESTfacility) RetrieveRegisteredMastodon() (instances []Provider, err error) { return nil, errors.New("not implemented") } // GetProviderOauthFor returns provider's params required for authenticated user to initiate an Oauth request // In case of Twitter, auth request url is fetched from twitter API endpoint on the fly. // For all requests, an Oauth session cache is initialized for requesting user, making use of cache facility. func (rest *RESTfacility) GetProviderOauthFor(userId, name, identifier string) (provider Provider, err CaliopenError) { provider, found := rest.providers[name] if found { switch provider.Name { case "twitter": requestToken, requestSecret, e := setTwitterAuthRequestUrl(&provider, rest.Hostname) if e != nil { log.WithError(e).Errorf("[GetProviderOauthFor] failed to set twitter auth request for user %s, provider %s", userId, name) err = WrapCaliopenErrf(e, FailDependencyCaliopenErr, "[GetProviderOauthFor] failed to set twitter auth request") return } cacheErr := rest.Cache.SetOauthSession(requestToken, &OauthSession{ RequestSecret: requestSecret, UserId: userId, }) if cacheErr != nil { log.WithError(cacheErr).Errorf("[GetProviderOauthFor] failed to set Oauth session in cache for user %s, provider %s", userId, name) err = WrapCaliopenErrf(e, FailDependencyCaliopenErr, "[GetProviderOauthFor] failed to set twitter Oauth session in cache") return } case "gmail": state, e := users.SetGoogleAuthRequestUrl(&provider, rest.Hostname) if e != nil { log.WithError(e).Errorf("[GetProviderOauthFor] failed to set gmail auth request for user %s, provider %s", userId, name) err = WrapCaliopenErrf(e, FailDependencyCaliopenErr, "[GetProviderOauthFor] failed to set gmail auth request") return } cacheErr := rest.Cache.SetOauthSession(state, &OauthSession{ UserId: userId, }) if cacheErr != nil { log.WithError(cacheErr).Errorf("[GetProviderOauthFor] failed to set Oauth session in cache for user %s, provider %s", userId, name) err = WrapCaliopenErrf(e, FailDependencyCaliopenErr, "[GetProviderOauthFor] failed to set gmail Oauth session in cache") return } case "mastodon": // identifier should be in the format `@username@server.tld` or `username@server.tld` identifier = strings.TrimPrefix(identifier, "@") account := strings.Split(identifier, "@") if len(account) != 2 || account[0] == "" || account[1] == "" { e := fmt.Errorf("[GetProviderOauthFor] missing or malformed mastodon identifier : <%s>", identifier) log.WithError(e) err = WrapCaliopenErrf(e, UnprocessableCaliopenErr, "[GetProviderOauthFor] failed to set mastodon auth request") return } u, e := url.Parse(account[1]) if e != nil || (u.Host == "" && u.Path == "") { log.WithError(e) err = WrapCaliopenErrf(e, UnprocessableCaliopenErr, "[GetProviderOauthFor] malformed mastodon instance : <%s>", account[1]) return } var instance string if u.Host != "" { instance = u.Host } else { instance = u.Path } // check if caliopen is already registered on instance mastoInstance, e := rest.store.RetrieveProvider("mastodon", instance) if e != nil || mastoInstance == nil { // else, register this caliopen app to mastodon instance app, e := mastodon.RegisterApp(context.Background(), &mastodon.AppConfig{ Server: "https://" + instance, ClientName: "Caliopen@" + rest.Hostname, Scopes: "read write follow", Website: "https://" + rest.Hostname, RedirectURIs: rest.Hostname + fmt.Sprintf(users.CALLBACK_BASE_URI, "mastodon"), }) if e != nil { err = WrapCaliopenErrf(e, FailDependencyCaliopenErr, "[GetProviderOauthFor] failed to register on mastodon instance <%s>", name) return } // => save instance's params into db for next time mastoInstance = new(Provider) mastoInstance.Name = name mastoInstance.Instance = instance mastoInstance.Protocol = "mastodon" mastoInstance.OauthRequestUrl = app.AuthURI mastoInstance.Infos = map[string]string{} mastoInstance.Infos["client_id"] = app.ClientID mastoInstance.Infos["client_secret"] = app.ClientSecret mastoInstance.Infos["address"] = "https://" + instance mastoInstance.Infos["auth_uri"] = app.AuthURI e = rest.store.CreateProvider(mastoInstance) if e != nil { log.WithError(e).Warnf("[GetProviderOauthFor] failed to save instance param in db for mastoInstance %+v", mastoInstance) } } else { mastoInstance.Protocol = "mastodon" mastoInstance.OauthRequestUrl = mastoInstance.Infos["auth_uri"] } state, e := users.SetMastodonAuthRequestUrl(mastoInstance, rest.Hostname) provider = *mastoInstance if e != nil { err = WrapCaliopenErrf(e, FailDependencyCaliopenErr, "[GetProviderOauthFor] failed to set mastodon auth request for user %s, on mastodon instance <%s>", userId, instance) return } cacheErr := rest.Cache.SetOauthSession(state, &OauthSession{ UserId: userId, Params: map[string]string{ "instance": instance, "account": account[0], }, }) if cacheErr != nil { log.WithError(cacheErr) err = WrapCaliopenErrf(e, FailDependencyCaliopenErr, "[GetProviderOauthFor] failed to set Oauth session in cache for user %s, provider %s", userId, name) return } default: err = NewCaliopenErr(NotImplementedCaliopenErr, "not implemented") return } return } log.Errorf("[GetProviderOauthFor] failed to found provider %s", name) err = NewCaliopenErr(NotFoundCaliopenErr, "not found") return } func (rest *RESTfacility) CreateTwitterIdentity(requestToken, verifier string) (remoteId string, err CaliopenError) { if requestToken == "" || verifier == "" { return "", NewCaliopenErr(UnprocessableCaliopenErr, "[CreateTwitterIdentity] missing oauth_token or oauth_verifier") } provider := rest.providers[TwitterProtocol] conf := &oauth1.Config{ ConsumerKey: provider.Infos["consumer_key"], ConsumerSecret: provider.Infos["consumer_secret"], Endpoint: twitterOAuth1.AuthorizeEndpoint, } oauthCache, e := rest.Cache.GetOauthSession(requestToken) if e != nil { log.WithError(e).Errorf("[CreateTwitterIdentity] failed to retrieve Oauth session in cache for request token %s and verifier %s", requestToken, verifier) err = WrapCaliopenErrf(e, NotFoundCaliopenErr, "[CreateTwitterIdentity] failed to retrieve Oauth session in cache for request token %s", requestToken) return } if len(oauthCache.RequestSecret) < 5 { log.WithError(e).Errorf("[CreateTwitterIdentity] oauthCache.RequestSecret too short to be a valid one: <%s> (token %s, verifier %s), user_id %s", oauthCache.RequestSecret, requestToken, verifier, oauthCache.UserId) err = WrapCaliopenErrf(e, FailDependencyCaliopenErr, "[CreateTwitterIdentity] invalid oauthCache.RequestSecret") return } accessToken, accessSecret, e := conf.AccessToken(requestToken, oauthCache.RequestSecret, verifier) if e != nil { log.WithError(e).Errorf("[CreateTwitterIdentity] failed to request an access token : token %s, verifier %s, secret : (5 first chars only) <%s>, user_id %s", requestToken, verifier, oauthCache.RequestSecret[0:5], oauthCache.UserId) err = WrapCaliopenErrf(e, FailDependencyCaliopenErr, "[CreateTwitterIdentity] AccessToken request failed") return } // retrieve twitter profile from Twitter api token := oauth1.NewToken(accessToken, accessSecret) httpClient := conf.Client(oauth1.NoContext, token) twitterClient := twitter.NewClient(httpClient) accountVerifyParams := &twitter.AccountVerifyParams{ IncludeEntities: twitter.Bool(false), SkipStatus: twitter.Bool(true), IncludeEmail: twitter.Bool(false), } twitterUser, resp, e := twitterClient.Accounts.VerifyCredentials(accountVerifyParams) if e != nil || resp.StatusCode != http.StatusOK || twitterUser == nil || twitterUser.ID == 0 || twitterUser.IDStr == "" { log.Errorf("[CreateTwitterIdentity] failed to get twitter user : error=%s, twitterUser=%+v, resp.StatusCode=%d", e, twitterUser, resp.StatusCode) err = NewCaliopenErr(FailDependencyCaliopenErr, "[CreateTwitterIdentity] twitter client failed to get Twitter User") return } // build user identity //1.check if this user_identity already exists foundIdentities, e := rest.store.LookupIdentityByIdentifier(twitterUser.ScreenName, TwitterProtocol) if e != nil { log.WithError(e).Errorf("[CreateTwitterIdentity] failed to lookup in store if identity already exists : screenName %s, protocol %s", twitterUser.ScreenName, TwitterProtocol) err = WrapCaliopenErrf(e, DbCaliopenErr, "[CreateTwitterIdentity] failed to lookup in store if identity already exists. Aborting") return } foundCount := len(foundIdentities) switch foundCount { case 0: userIdentity := new(UserIdentity) userID := UUID(uuid.FromStringOrNil(oauthCache.UserId)) userIdentity.MarshallNew(userID) userIdentity.Protocol = TwitterProtocol userIdentity.Type = RemoteIdentity userIdentity.DisplayName = twitterUser.Name userIdentity.Identifier = twitterUser.ScreenName userIdentity.Credentials = &Credentials{ "token": accessToken, "secret": accessSecret, } userIdentity.Infos = map[string]string{ "provider": "twitter", "authtype": Oauth1, "twitterid": twitterUser.IDStr, } // save identity e := rest.CreateUserIdentity(userIdentity) if e != nil { log.WithError(e).Errorf("[CreateTwitterIdentity] failed to create user identity : %+v", *userIdentity) err = WrapCaliopenErr(e, FailDependencyCaliopenErr, "[CreateTwitterIdentity] failed to create user identity") return } remoteId = userIdentity.Id.String() return case 1: // this twitter identity already exists, checking if it belongs to this user and, if ok, just updating Twitter's name storedIdentity, e := rest.RetrieveUserIdentity(foundIdentities[0][0], foundIdentities[0][1], false) if e != nil || storedIdentity == nil { log.Errorf("[CreateTwitterIdentity] failed to retrieve user identity found for twitter account %s. Error=%s, identity=%+v", twitterUser.ScreenName, e, *storedIdentity) err = WrapCaliopenErrf(e, DbCaliopenErr, "[CreateTwitterIdentity] failed to retrieve user identity found for twitter account %s", twitterUser.ScreenName) return } modifiedFields := map[string]interface{}{ "DisplayName": twitterUser.Name, } storedIdentity.DisplayName = twitterUser.Name if e := rest.store.UpdateUserIdentity(storedIdentity, modifiedFields); e != nil { log.WithError(e).Errorf("[CreateTwitterIdentity] failed to update user identity in db : identity=%+v, fields=%+v", *storedIdentity, modifiedFields) err = WrapCaliopenErrf(e, FailDependencyCaliopenErr, "[CreateTwitterIdentity] failed to update user identity in db") return } remoteId = storedIdentity.Id.String() return default: log.Errorf("[CreateTwitterIdentity] inconsistency in store : more than one identity found with twitter screen name <%s>", twitterUser.ScreenName) err = NewCaliopenErrf(FailDependencyCaliopenErr, "[CreateTwitterIdentity] inconsistency in store : more than one identity found with twitter screen name <%s>", twitterUser.ScreenName) return } } func (rest *RESTfacility) CreateGmailIdentity(state, code string) (remoteId string, err CaliopenError) { // start by checking if we have the state in cache oauthCache, e := rest.Cache.GetOauthSession(state) if e != nil { log.WithError(e).Errorf("[CreateGmailIdentity] failed to retrieve Oauth session in cache for state %s", state) err = WrapCaliopenErrf(e, NotFoundCaliopenErr, "[CreateGmailIdentity] failed to retrieve Oauth session in cache for state %s", state) return } gmailProvider := rest.providers["gmail"] gmailProvider.OauthCallbackUri = fmt.Sprintf(users.CALLBACK_BASE_URI, "gmail") oauthConfig := users.SetGoogleOauthConfig(gmailProvider, rest.Hostname) ctx := context.Background() // get access & refresh tokens that will be added to UserIdentity.Credentials token, e := oauthConfig.Exchange(ctx, code, oauth2.AccessTypeOffline) if e != nil { log.WithError(e).Errorf("[CreateGmailIdentity] failed to exchange access token for state %s and code %s. oauthConfig=%+v", state, code, *oauthConfig) err = WrapCaliopenErrf(e, NotFoundCaliopenErr, "[CreateGmailIdentity] failed to retrieve access token for state %s", state) return } // retrieve gmail profile from Google api httpClient := oauthConfig.Client(ctx, token) googleService, e := googleApi.New(httpClient) if e != nil { log.WithError(e).Errorf("[CreateGmailIdentity] failed to create google service with oauthconfig=%+v, httpclient=%+v", *oauthConfig, *httpClient) err = WrapCaliopenErr(e, NotFoundCaliopenErr, "[CreateGmailIdentity] failed to create google service") return } googleUser, e := googleService.Userinfo.Get().Do() if e != nil { log.WithError(e).Errorf("[CreateGmailIdentity] failed to retrieve user from google api, googleService=%+v", *googleService) err = WrapCaliopenErr(e, NotFoundCaliopenErr, "[CreateGmailIdentity] failed to retrieve user from google api") return } // build user identity // 1. check if this user_identity already exists foundIdentities, e := rest.store.LookupIdentityByIdentifier(googleUser.Email, EmailProtocol) if e != nil { log.WithError(e).Errorf("[CreateGmailIdentity] failed to lookup identity in store : googleUser.Email=%s, protocol=%s", googleUser.Email, EmailProtocol) err = WrapCaliopenErrf(e, DbCaliopenErr, "[CreateGmailIdentity] failed to lookup in store if identity already exists. Aborting") return } foundCount := len(foundIdentities) switch foundCount { case 0: // it appears that google API does not return a refresh token if Caliopen has already been authorize for this google account and user is logged in his google account if token.RefreshToken == "" { log.WithError(e).Errorf("[CreateGmailIdentity] exchange access token for state %s and code %s got an empty refreshToken. oauthConfig=%+v, token=%+v", state, code, *oauthConfig, *token) err = WrapCaliopenErrf(e, NotFoundCaliopenErr, "[CreateGmailIdentity] failed to get a refresh token for state %s. User should check application permissions.", state) return } userIdentity := new(UserIdentity) userID := UUID(uuid.FromStringOrNil(oauthCache.UserId)) userIdentity.MarshallNew(userID) userIdentity.Protocol = EmailProtocol userIdentity.Type = RemoteIdentity userIdentity.DisplayName = googleUser.Name userIdentity.Identifier = googleUser.Email userIdentity.Credentials = &Credentials{ users.CRED_ACCESS_TOKEN: token.AccessToken, users.CRED_REFRESH_TOKEN: token.RefreshToken, users.CRED_TOKEN_EXPIRY: token.Expiry.Format(time.RFC3339), users.CRED_TOKEN_TYPE: token.TokenType, users.CRED_USERNAME: googleUser.Email, } userIdentity.Infos = map[string]string{ "inserver": gmailProvider.Infos["imapserver"], "outserver": gmailProvider.Infos["smtpserver"], "provider": "gmail", "authtype": Oauth2, } // save identity e := rest.CreateUserIdentity(userIdentity) if e != nil { log.WithError(e).Errorf("[CreateGmailIdentity] failed to create user identity : userIdentity=%+v", *userIdentity) err = WrapCaliopenErr(e, FailDependencyCaliopenErr, "[CreateGmailIdentity] failed to create user identity") return } remoteId = userIdentity.Id.String() case 1: // this identity already exists, // WHAT TO DO ?? -> stop here or continue ? // checking if it belongs to this user and, if ok, just updating display name // but not tokens storedIdentity, e := rest.RetrieveUserIdentity(foundIdentities[0][0], foundIdentities[0][1], false) if e != nil || storedIdentity == nil { log.WithError(e).Errorf("[CreateGmailIdentity] failed to retrieve user identity found for google account %s: foundIdentities=%+v", googleUser.Name, foundIdentities) err = WrapCaliopenErrf(e, DbCaliopenErr, "[CreateGmailIdentity] failed to retrieve user identity found for google account %s", googleUser.Name) return } storedIdentity.DisplayName = googleUser.Name /* storedIdentity.Credentials = &Credentials{ users.CRED_ACCESS_TOKEN: token.AccessToken, users.CRED_REFRESH_TOKEN: token.RefreshToken, users.CRED_TOKEN_EXPIRY: token.Expiry.Format(time.RFC3339), users.CRED_TOKEN_TYPE: token.TokenType, users.CRED_USERNAME: googleUser.Email, } */ modifiedFields := map[string]interface{}{ "DisplayName": googleUser.Name, //"Credentials": storedIdentity.Credentials, } if e := rest.store.UpdateUserIdentity(storedIdentity, modifiedFields); e != nil { log.WithError(e).Errorf("[CreateGmailIdentity] failed to update user identity in db : identity=%+v, fields=%+v", *storedIdentity, modifiedFields) err = WrapCaliopenErrf(e, FailDependencyCaliopenErr, "[CreateGmailIdentity] failed to update user identity in db") return } remoteId = storedIdentity.Id.String() return default: log.Errorf("[CreateGmailIdentity] inconsistency in store : more than one identity found with email <%s>. foundIdentities=%+v", googleUser.Email, foundIdentities) err = NewCaliopenErrf(FailDependencyCaliopenErr, "[CreateGmailIdentity] inconsistency in store : more than one identity found with email <%s>", googleUser.Email) return } return } func (rest *RESTfacility) CreateMastodonIdentity(state, code string) (remoteId string, err CaliopenError) { // start by checking if we have the state in cache oauthCache, e := rest.Cache.GetOauthSession(state) if e != nil { log.WithError(e).Errorf("[CreateMastodonIdentity] failed to retrieve Oauth session in cache for state %s", state) err = WrapCaliopenErrf(e, NotFoundCaliopenErr, "[CreateMastodonIdentity] failed to retrieve Oauth session in cache for state %s", state) return } mastodonProvider, e := rest.store.RetrieveProvider("mastodon", oauthCache.Params["instance"]) if e != nil || mastodonProvider == nil { log.WithError(e) err = WrapCaliopenErrf(e, FailDependencyCaliopenErr, "[CreateMastodonIdentity] failed to retrieve mastodon instance %s from db", oauthCache.Params["instance"]) return } mastodonProvider.OauthCallbackUri = fmt.Sprintf(users.CALLBACK_BASE_URI, "mastodon") ctx := context.Background() // get access token that will be added to UserIdentity.Credentials oauthConfig := users.SetMastodonOauthConfig(*mastodonProvider, rest.Hostname) token, e := oauthConfig.Exchange(ctx, code, oauth2.AccessTypeOffline) if e != nil { log.WithError(e).Errorf("[CreateMastodonIdentity] failed to exchange access token for state %s and code %s. oauthConfig=%+v", state, code, *oauthConfig) err = WrapCaliopenErrf(e, NotFoundCaliopenErr, "[CreateMastodonIdentity] failed to retrieve access token for state %s", state) return } // retrieve user profile from mastodon API mastodonClient := mastodon.NewClient(&mastodon.Config{ Server: mastodonProvider.Infos["address"], ClientID: mastodonProvider.Infos["client_id"], ClientSecret: mastodonProvider.Infos["client_secret"], AccessToken: token.AccessToken, }) userAcct, acctErr := mastodonClient.GetAccountCurrentUser(ctx) if acctErr != nil { log.WithError(acctErr) err = WrapCaliopenErrf(acctErr, FailDependencyCaliopenErr, "[CreateMastodonIdentity] failed to retrieve mastodon user account") return } // build user identity // 1. check if this user_identity already exists mastodonIdentifier := userAcct.Username + "@" + mastodonProvider.Instance foundIdentities, e := rest.store.LookupIdentityByIdentifier(mastodonIdentifier, MastodonProtocol) if e != nil { log.WithError(e).Errorf("[CreateMastodonIdentity] failed to lookup mastodon identity in store : %s", userAcct.Acct) err = WrapCaliopenErrf(e, DbCaliopenErr, "[CreateMastodonIdentity] failed to lookup in store if identity already exists. Aborting") return } foundCount := len(foundIdentities) switch foundCount { case 0: userIdentity := new(UserIdentity) userID := UUID(uuid.FromStringOrNil(oauthCache.UserId)) userIdentity.MarshallNew(userID) userIdentity.Protocol = MastodonProtocol userIdentity.Type = RemoteIdentity userIdentity.DisplayName = userAcct.DisplayName userIdentity.Identifier = mastodonIdentifier userIdentity.Credentials = &Credentials{ users.CRED_ACCESS_TOKEN: token.AccessToken, users.CRED_TOKEN_EXPIRY: token.Expiry.Format(time.RFC3339), users.CRED_TOKEN_TYPE: token.TokenType, } userIdentity.Infos = map[string]string{ "provider": "mastodon", "authtype": Oauth2, "mastodon_id": string(userAcct.ID), "url": userAcct.URL, "avatar": userAcct.Avatar, } // save identity e := rest.CreateUserIdentity(userIdentity) if e != nil { log.WithError(e).Errorf("[CreateMastodonIdentity] failed to create user identity : userIdentity=%+v", *userIdentity) err = WrapCaliopenErr(e, FailDependencyCaliopenErr, "[CreateMastodonIdentity] failed to create user identity") return } remoteId = userIdentity.Id.String() case 1: // this identity already exists, // WHAT TO DO ?? -> stop here or continue ? // checking if it belongs to this user and, if ok, just updating display name // but not tokens storedIdentity, e := rest.RetrieveUserIdentity(foundIdentities[0][0], foundIdentities[0][1], false) if e != nil || storedIdentity == nil { log.WithError(e).Errorf("[CreateMastodonIdentity] failed to retrieve user identity found for account %s: foundIdentities=%+v", mastodonIdentifier, foundIdentities) err = WrapCaliopenErrf(e, DbCaliopenErr, "[CreateMastodonIdentity] failed to retrieve user identity found for account %s", mastodonIdentifier) return } storedIdentity.DisplayName = userAcct.DisplayName modifiedFields := map[string]interface{}{ "DisplayName": userAcct.DisplayName, } if e := rest.store.UpdateUserIdentity(storedIdentity, modifiedFields); e != nil { log.WithError(e).Errorf("[CreateMastodonIdentity] failed to update user identity in db : identity=%+v, fields=%+v", *storedIdentity, modifiedFields) err = WrapCaliopenErrf(e, FailDependencyCaliopenErr, "[CreateMastodonIdentity] failed to update user identity in db") return } remoteId = storedIdentity.Id.String() return default: log.Errorf("[CreateMastodonIdentity] inconsistency in store : more than one identity found with name <%s>. foundIdentities=%+v", mastodonIdentifier, foundIdentities) err = NewCaliopenErrf(FailDependencyCaliopenErr, "[CreateMastodonIdentity] inconsistency in store : more than one identity found with name <%s>", mastodonIdentifier) return } return } func setTwitterAuthRequestUrl(provider *Provider, hostname string) (requestToken, requestSecret string, err CaliopenError) { provider.OauthCallbackUri = fmt.Sprintf(users.CALLBACK_BASE_URI, "twitter") //IMPORTANT TODO: make use of vault to store consumer key and secret conf := &oauth1.Config{ ConsumerKey: provider.Infos["consumer_key"], ConsumerSecret: provider.Infos["consumer_secret"], CallbackURL: hostname + provider.OauthCallbackUri, Endpoint: twitterOAuth1.AuthorizeEndpoint, } requestToken, requestSecret, e := conf.RequestToken() if e != nil { log.WithError(e).Errorf("[setTwitterAuthRequestUrl] failed to request token with config : %+v", *conf) err = WrapCaliopenErrf(e, FailDependencyCaliopenErr, "[setTwitterAuthRequestUrl] failed with RequestToken()") return } authUrl, e := conf.AuthorizationURL(requestToken) if e != nil { log.WithError(e).Errorf("[setTwitterAuthRequestUrl] failed to request auth url with config : %+v, token %s", *conf, requestToken) err = WrapCaliopenErrf(e, FailDependencyCaliopenErr, "[setTwitterAuthRequestUrl] failed with AuthorizationURL()") return } provider.OauthRequestUrl = authUrl.String() return } ================================================ FILE: src/backend/main/go.main/facilities/REST/search.go ================================================ package REST import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" m "github.com/CaliOpen/Caliopen/src/backend/main/go.main/messages" ) // API to execute broad-based searches within index // Searches are executed in all docs types of user func (rest *RESTfacility) Search(search IndexSearch) (response *IndexResult, err error) { // double check search object consistency before triggering the search if search.DocType != "" { if search.DocType != MessageIndexType && search.DocType != ContactIndexType { return nil, errors.New("[RESTfacility] Invalid doc_type in search request") } } else { if search.Offset > 0 || search.Limit > 0 { return nil, errors.New("[RESTfacility] invalid search request: params 'offset', 'limit' are only accepted if param 'doc_type' is also provided") } } // trigger the search result, err := rest.index.Search(search) if err != nil { return nil, err } // prepare messages objects for frontend rendering for _, doc := range result.MessagesHits.Messages { msg := doc.Document.(*Message) m.SanitizeMessageBodies(msg) (*msg).Body_excerpt = m.ExcerptMessage(*msg, 200, true, true) } return result, nil } ================================================ FILE: src/backend/main/go.main/facilities/REST/settings.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package REST import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) func (rest *RESTfacility) GetSettings(user_id string) (settings *Settings, err error) { settings, err = rest.store.GetSettings(user_id) if err != nil { return nil, err } return settings, err } ================================================ FILE: src/backend/main/go.main/facilities/REST/suggest_participants.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package REST import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) // SuggestRecipients makes use of index facility to return to user a list of suggested recipients // within the context of composing a new message // list is ordered by relevance : first suggestion should be the best func (rest *RESTfacility) SuggestRecipients(user *UserInfo, query_string string) (suggests []RecipientSuggestion, err error) { if user != nil && query_string != "" && len(query_string) > 2 { // TODO : more consistency checking against user_id & query_string return rest.index.RecipientsSuggest(user, query_string) } else { err = errors.New("[RESTfacility.SuggestRecipients] unprocessable parameters") return } } ================================================ FILE: src/backend/main/go.main/facilities/REST/tags.go ================================================ package REST import ( "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/helpers" "github.com/bitly/go-simplejson" "github.com/mozillazg/go-unidecode" "strings" ) func (rest *RESTfacility) RetrieveUserTags(user_id string) (tags []Tag, err CaliopenError) { tags, e := rest.store.RetrieveUserTags(user_id) if e != nil { return tags, WrapCaliopenErr(e, DbCaliopenErr, "[RESTfacility] RetrieveUserTags failed") } return tags, nil } // CreateTag : // - ensures tag's label is unique // - copies tag's name to tag's label // - converts tag's name to lower & ASCII and replace spaces by "_" // - adds the tag in db for user if it doesn't exist yet // - updates tag in-place with its new properties func (rest *RESTfacility) CreateTag(tag *Tag) CaliopenError { var isUnique bool var err error isUnique, tag.Name, err = rest.IsTagLabelNameUnique(tag.Label, tag.User_id.String()) if err != nil { return WrapCaliopenErr(err, UnprocessableCaliopenErr, "[RESTfacility] CreateTag failed to check tag's uniqueness") } if !isUnique { return NewCaliopenErr(UnprocessableCaliopenErr, "[RESTfacility] tag's name/label conflict with existing one") } err = rest.store.CreateTag(tag) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] CreateTag failed to CreateTag in store") } return nil } func (rest *RESTfacility) RetrieveTag(user_id, tag_name string) (tag Tag, err CaliopenError) { tag, e := rest.store.RetrieveTag(user_id, tag_name) if e != nil { return tag, WrapCaliopenErr(e, DbCaliopenErr, "[RESTfacility] RetrieveTag failed") } return tag, nil } // PatchTag is a shortcut for REST api to call two methods : // - UpdateWithPatch () to retrieve the tag from db // - checks that new label is not conflicting with an existing one // - then UpdateTag() to save updated tag to stores if everything went good. func (rest *RESTfacility) PatchTag(patch []byte, user_id, tag_name string) CaliopenError { current_tag, Cerr := rest.RetrieveTag(user_id, tag_name) if Cerr != nil { return Cerr } // read into the patch to make basic controls before processing it with generic helper patchReader, err := simplejson.NewJson(patch) if err != nil { return WrapCaliopenErrf(err, FailDependencyCaliopenErr, "[RESTfacility] PatchTag failed with simplejson error : %s", err) } label := patchReader.Get("label").MustString() if label == "" || strings.Replace(label, " ", "", -1) == "" { return NewCaliopenErr(UnprocessableCaliopenErr, "[RESTfacility] new tag's label is empty") } isUnique, name, err := rest.IsTagLabelNameUnique(label, user_id) if err != nil { return NewCaliopenErr(UnprocessableCaliopenErr, "[RESTfacility] tag's name/label conflict with existing one") } if !isUnique && name != tag_name { return NewCaliopenErr(UnprocessableCaliopenErr, "[RESTfacility] tag's name/label conflict with existing one") } // patch seams OK, apply it to the resource newTag, _, err := helpers.UpdateWithPatch(patch, ¤t_tag, UserActor) if err != nil { return WrapCaliopenErrf(err, FailDependencyCaliopenErr, "[RESTfacility] PatchTag failed with UpdateWithPatch error : %s", err) } // save updated resource err = rest.UpdateTag(newTag.(*Tag)) if err != nil { return WrapCaliopenErrf(err, FailDependencyCaliopenErr, "[RESTfacility] PatchTag failed with UpdateTag error : %s", err) } return nil } // UpdateTag updates a tag in store with payload, func (rest *RESTfacility) UpdateTag(tag *Tag) CaliopenError { user_id := tag.User_id.String() if user_id != "" && tag.Name != "" { db_tag, err := rest.store.RetrieveTag(user_id, tag.Name) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] UpdateTag failed to RetrieveTag from store") } // RESTfacility allows user to only modify label and importance_level properties // thus squash other properties with those from db to ignore any modifications tag.Date_insert = db_tag.Date_insert tag.Name = db_tag.Name tag.Type = db_tag.Type tag.User_id = db_tag.User_id err = rest.store.UpdateTag(tag) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] UpdateTag failed to UpdateTag in store") } } else { return NewCaliopenErr(UnprocessableCaliopenErr, "[RESTfacility] invalid tag's name and/or user_id") } return nil } // DeleteTag deletes a tag in store, // only if tag is a user tag. func (rest *RESTfacility) DeleteTag(user_id, tag_name string) CaliopenError { tag, err := rest.store.RetrieveTag(user_id, tag_name) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] DeleteTag failed to retrieve tag") } if tag.Type == SystemTag { return NewCaliopenErr(UnprocessableCaliopenErr, "[RESTfacility] system tags can't be deleted by user") } err = rest.store.DeleteTag(user_id, tag_name) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] DeleteTag failed to DeleteTag in store") } go rest.deleteEmbeddedTagReferences(user_id, tag_name) return nil } // UpdateResourceTags : // - checks that tag_names within patch belong to user and are unique, // - calls generic UpdateWithPatch func to patch the resource, // - saves and indexes updated resource. // It is caller responsibility to call this func with a well-formed patch that has only "tags" properties func (rest *RESTfacility) UpdateResourceTags(user *UserInfo, resourceID, resourceType string, patch []byte) CaliopenError { var err error // 1. check that tag_names within patch belong to user and are unique tags, err := rest.RetrieveUserTags(user.User_id) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] UpdateResourceTags error") } userTagsMap := make(map[string]bool) for _, tag := range tags { userTagsMap[tag.Name] = true } p, err := simplejson.NewJson(patch) if err != nil { return WrapCaliopenErr(err, UnknownCaliopenErr, "[RESTfacility] UpdateResourceTags simplejson error") } deduplicatedTagsList := []string{} patchTagsMap := make(map[string]bool) for _, tag := range p.Get("tags").MustStringArray() { // is tag belonging to user ? if _, ok := userTagsMap[tag]; !ok { err = fmt.Errorf("[RESTfacility] UpdateResourceTags : tag with name <%s> does not belong to user <%s>", tag, user.User_id) break } // is tag unique ? if _, ok := patchTagsMap[tag]; !ok { patchTagsMap[tag] = true deduplicatedTagsList = append(deduplicatedTagsList, tag) } } if err != nil { return WrapCaliopenErr(err, UnknownCaliopenErr, "[RESTfacility] UpdateResourceTags") } p.Set("tags", deduplicatedTagsList) patch, _ = p.MarshalJSON() // 2. call generic UpdateWitchPatch func var obj ObjectPatchable var newObj ObjectPatchable switch resourceType { case MessageType: m, err := rest.store.RetrieveMessage(user.User_id, resourceID) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] UpdateResourceTags") } obj = ObjectPatchable(m) case ContactType: c, err := rest.store.RetrieveContact(user.User_id, resourceID) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] UpdateResourceTags") } obj = ObjectPatchable(c) default: return NewCaliopenErr(UnknownCaliopenErr, "[RESTfacility] UpdateResourceWithPatch : invalid resourceType") } newObj, _, err = helpers.UpdateWithPatch(patch, obj, SystemActor) if err != nil { return WrapCaliopenErr(err, UnknownCaliopenErr, "[RESTfacility] UpdateResourceTags : helpers.UpdateWithPatch failed") } // 3. store and index updated resource switch resourceType { case MessageType: update := map[string]interface{}{ "Tags": newObj.(*Message).Tags, } err := rest.store.UpdateMessage(newObj.(*Message), update) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] UpdateResourceTags") } err = rest.index.UpdateMessage(user, obj.(*Message), update) if err != nil { return WrapCaliopenErr(err, IndexCaliopenErr, "[RESTfacility] UpdateResourceTags") } case ContactType: update := map[string]interface{}{ "Tags": newObj.(*Contact).Tags, } err := rest.store.UpdateContact(newObj.(*Contact), obj.(*Contact), update) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] UpdateResourceTags") } err = rest.index.UpdateContact(user, newObj.(*Contact), update) if err != nil { return WrapCaliopenErr(err, IndexCaliopenErr, "[RESTfacility] UpdateResourceTags") } } return nil } // IsTagLabelNameUnique ensures that a name and/or label is not conflicting with a name/label of another tag. // returns : // - true if name/label is unique, along with the labelname normalized to a string that could be used as new tag's slug. // - false if name/label already exists, along with the slug that conflict with func (rest *RESTfacility) IsTagLabelNameUnique(labelname, userID string) (bool, string, error) { slug := utf8ToASCIILowerNoSpace(labelname) tags, err := rest.store.RetrieveUserTags(userID) if err != nil { return false, "", err } for _, t := range tags { if t.Name == slug || t.Label == slug || t.Label == labelname { return false, slug, nil } } return true, slug, nil } func (rest *RESTfacility) deleteEmbeddedTagReferences(userID, tagName string) { //TODO : async deletion of tag references embedded into resources. } func utf8ToASCIILowerNoSpace(s string) string { b := make([]byte, len(s)) b = []byte(unidecode.Unidecode(s)) b = []byte(strings.ToLower(string(b))) b = []byte(strings.Replace(string(b), " ", "_", -1)) return string(b) } ================================================ FILE: src/backend/main/go.main/facilities/REST/username.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package REST func (rest *RESTfacility) UsernameIsAvailable(username string) (bool, error) { return rest.store.UsernameIsAvailable(username) } ================================================ FILE: src/backend/main/go.main/facilities/REST/users.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package REST import ( "errors" "fmt" "time" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/Notifications" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/users" "github.com/Sirupsen/logrus" "github.com/renstrom/shortuuid" "github.com/satori/go.uuid" "github.com/tidwall/gjson" "golang.org/x/crypto/bcrypt" ) const ( changePasswordSubject = "Information Caliopen : votre mot de passe a été changé" changePasswordBodyPlain = ` Caliopen vous informe que le mot de passe de votre compte a été changé. ` changePasswordBodyRich = changePasswordBodyPlain ) // as of oct. 2017, PatchUser only implemented for changing user's password // any attempt to patch something else should trigger an error func (rest *RESTfacility) PatchUser(userId string, patch *gjson.Result, notify Notifications.Notifiers) error { user, err := rest.store.RetrieveUser(userId) if err != nil { return err } if patch.Get("password").Exists() { // if found a `password` property in patch, then special case : // there should be no other properties to patch err = validatePasswordPatch(patch) if err != nil { return errors.New("[REST PatchUser] invalid password patch : " + err.Error()) } // call the service that change user password err = users.ChangeUserPassword(user, patch, rest.store) if err != nil { return err } // compose and send a notification via email notif := &Notification{ Body: fmt.Sprintf("password changed for user %s", user.UserId.String()), InternalPayload: &Message{ Body_plain: changePasswordBodyPlain, Body_html: changePasswordBodyRich, Subject: changePasswordSubject, }, NotifId: UUID(uuid.NewV1()), Type: NotifAdminMail, User: user, } go notify.ByEmail(notif) return nil } else { // hack to ensure that patch is for password only // should be replaced by : // helpers.ValidatePatchSemantic(user, patch) return errors.New("[REST] PatchUser only implemented for changing password (for now)") } return nil } func (rest *RESTfacility) GetUser(userId string) (user *User, err error) { //TODO return } // validatePasswordPatch checks if patch has only `password` property func validatePasswordPatch(patch *gjson.Result) error { var err error current_state := patch.Get("current_state") if !current_state.Exists() { return errors.New("[Patch] missing 'current_state' property in patch json") } if !current_state.IsObject() { return errors.New("[Patch] 'current_state' property in patch json is not an object") } keyValidator := func(key, value gjson.Result) bool { if key.String() != "current_state" && key.String() != "password" { err = errors.New(fmt.Sprintf("[Patch] found invalid key <%s> in the password patch", key.String())) return false } return true } patch.ForEach(keyValidator) if err == nil { current_state.ForEach(keyValidator) } return err } // RequestPasswordReset checks if an user could be found with provided payload request, // if found, it will trigger the password reset procedure that ends by notifying the user via the provided notifiers interface func (rest *RESTfacility) RequestPasswordReset(payload PasswordResetRequest, notify Notifications.Notifiers) error { var user *User var err error // 1. check if user exist if payload.Username != "" { user, err = rest.store.UserByUsername(payload.Username) if err != nil || user == nil { logrus.Info(err) return errors.New("[RESTfacility] user not found") } if payload.RecoveryMail != "" { // check if provided email is consistent for this user if payload.RecoveryMail != user.RecoveryEmail { return errors.New("[RESTfacility] username and recovery email mismatch") } } } else if payload.RecoveryMail != "" { user, err = rest.store.UserByRecoveryEmail(payload.RecoveryMail) if err != nil || user == nil { logrus.Info(err) return errors.New("[RESTfacility] user not found") } if payload.Username != "" { // check if provided username is consistent for this user if payload.Username != user.Name { return errors.New("[RESTfacility] username and recovery email mismatch") } } } else { return errors.New("[RESTfacility] neither username, nor recovery email provided, at least one required") } // 2. check if a password reset has already been ignited for that user reset_session, err := rest.Cache.GetResetPasswordSession(user.UserId.String()) if reset_session != nil { rest.Cache.DeleteResetPasswordSession(user.UserId.String()) logrus.Infof("[RESTFacility] reset password session deleted for user <%s> [%s]", user.Name, user.UserId.String()) } // 3. generate a reset token and cache it token := shortuuid.New() reset_session, err = rest.Cache.SetResetPasswordSession(user.UserId.String(), token) if err != nil { return err } // 4. send reset link to user's recovery email address. notif := &Notification{ User: user, InternalPayload: reset_session, NotifId: UUID(uuid.NewV1()), Body: fmt.Sprintf("reset link for user %s", user.UserId.String()), Type: NotifPasswordReset, } go notify.ByEmail(notif) logrus.Infof("[RESTFacility] reset password session ignited for user <%s> [%s]", user.Name, user.UserId.String()) return nil } func (rest *RESTfacility) ValidatePasswordResetToken(token string) (session *TokenSession, err error) { session, err = rest.Cache.GetResetPasswordToken(token) if err != nil || session == nil { return nil, errors.New("[RESTfacility] token not found") } if time.Now().After(session.ExpiresAt) { // unlikely to happen because ttl is also set in cache facility return nil, errors.New("[RESTfacility] token expired") } return session, nil } func (rest *RESTfacility) ResetUserPassword(token, new_password string, notify Notifications.Notifiers) error { session, err := rest.ValidatePasswordResetToken(token) if err != nil { return err } user, err := rest.store.RetrieveUser(session.UserId) if err != nil { return err } // reset password err = users.ResetUserPassword(user, new_password, rest.store) if err != nil { return err } logrus.Infof("[RESTFacility] password reset for user <%s> [%s]", user.Name, user.UserId.String()) // delete reset session cache err = rest.Cache.DeleteResetPasswordSession(user.UserId.String()) if err != nil { logrus.WithError(err).Warnf("[RESTfacility] failed to delete reset session cache for user %s", user.UserId.String()) } else { logrus.Infof("[RESTFacility] reset password session deleted for user <%s> [%s]", user.Name, user.UserId.String()) } // send email notification to user's recovery email address notif := Notification{ User: user, InternalPayload: &Message{ Body_plain: changePasswordBodyPlain, Body_html: changePasswordBodyRich, Subject: changePasswordSubject, }, NotifId: UUID(uuid.NewV1()), Body: fmt.Sprintf("password changed for user %s", user.UserId.String()), Type: NotifAdminMail, } go notify.ByEmail(¬if) return nil } // DeleteUser deletes a user in store, // Check the password as a validation before func (rest *RESTfacility) DeleteUser(payload ActionsPayload) CaliopenError { if payload.Params == nil { return NewCaliopenErr(UnprocessableCaliopenErr, "[RESTfacility] delete user params is missing") } if params, ok := payload.Params.(DeleteUserParams); ok { user, err := rest.store.RetrieveUser(payload.UserId) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] DeleteUser failed to retrieve user") } if !user.DateDelete.IsZero() { return NewCaliopenErr(UnprocessableCaliopenErr, "[RESTfacility] User already deleted.") } err = bcrypt.CompareHashAndPassword(user.Password, []byte(params.Password)) if err != nil { return WrapCaliopenErr(err, WrongCredentialsErr, "[RESTfacility] DeleteUser Wrong password") } err = rest.store.DeleteUser(payload.UserId) if err != nil { return WrapCaliopenErr(err, DbCaliopenErr, "[RESTfacility] DeleteUser failed to delete user in store") } // Logout err = rest.Cache.LogoutUser(params.AccessToken) if err != nil { return NewCaliopenErr(UnprocessableCaliopenErr, "[RESTfacility] Unable to logout.") } } else { return NewCaliopenErr(UnprocessableCaliopenErr, "[RESTfacility] payload.Params is not of type DeleteUserParams") } return nil } ================================================ FILE: src/backend/main/go.main/helpers/contact.go ================================================ /* * // Copyleft (ɔ) 2017 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package helpers import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/ttacon/libphonenumber" "strings" ) // ComputeNewTitle applies ComputeTitle logic to a new Contact // If names are empty but Title is not, names are filled from Title. func ComputeNewTitle(c *Contact) { // TODO: improve it if c.Title == "" { ComputeTitle(c) } else { // Title has been set by user, // if given_name & family_name are empty we try to fill them from title if c.GivenName == "" && c.FamilyName == "" { if space_sep := strings.Split(c.Title, " "); len(space_sep) > 1 { //fill it the Google way : last word to family_name c.FamilyName = space_sep[len(space_sep)-1] c.GivenName = strings.Join(space_sep[:len(space_sep)-1], " ") } else if comma_sep := strings.Split(c.Title, ","); len(comma_sep) > 1 { //same algo as above, but with comma c.FamilyName = space_sep[len(space_sep)-1] c.GivenName = strings.Join(space_sep[:len(space_sep)-1], " ") c.Title = strings.Replace(c.Title, ",", " ", -1) } else { c.FamilyName = c.Title } } } } // ComputeTitle modifies Title property in-place or do nothing if it fails // it tries to fill Title with names, then email/phones/etc. func ComputeTitle(c *Contact) { title := []string{} if c.NamePrefix != "" { title = append(title, c.NamePrefix) } if c.GivenName != "" { title = append(title, c.GivenName) } if c.AdditionalName != "" { title = append(title, c.AdditionalName) } if c.FamilyName != "" { title = append(title, c.FamilyName) } if c.NameSuffix != "" { title = append(title, c.NameSuffix) } switch len(title) { case 0: //Title is still empty. try more properties switch { case len(c.Emails) > 0: c.Title = c.Emails[0].Address case len(c.Phones) > 0: c.Title = c.Phones[0].Number case len(c.Ims) > 0: c.Title = c.Ims[0].Address } case 1: c.Title = title[0] default: c.Title = strings.Join(title, " ") } } // NormalizePhoneNumbers tries to normalize phone numbers found within contact. // It fills Phone.NormalizedNumber property if it could func NormalizePhoneNumbers(c *Contact) { var num libphonenumber.PhoneNumber for i, phone := range c.Phones { // try to parse phone number by seeking a country code // fallback to french number only if no country code could be found // TODO: lookup into user's contactbook to find the most relevant country code to fallback to if err := libphonenumber.ParseToNumber(phone.Number, "FR", &num); err == nil { c.Phones[i].NormalizedNumber = libphonenumber.Format(&num, libphonenumber.INTERNATIONAL) } } } ================================================ FILE: src/backend/main/go.main/helpers/discussion.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package helpers import ( log "github.com/Sirupsen/logrus" "github.com/montanaflynn/stats" ) // ComputeDiscussionIL is the algorithm to return an Importance Level for a discussion // based on importance levels found within messages of the discussion. // It returns the average Importance Level of the top 10% most important messages. // If calculation fails, returned 0. func ComputeDiscussionIL(messagesIL []float64) float64 { rank, err := stats.PercentileNearestRank(messagesIL, 90) if err != nil { log.WithError(err).Error("[ComputeDiscussionIL] failed to get percentile from set") return 0 } var percentile stats.Float64Data for _, n := range messagesIL { if n >= rank { percentile = append(percentile, n) } } mean, err := stats.Mean(percentile) if err != nil { log.WithError(err).Error("[ComputeDiscussionIL] failed to compute mean from set") return 0 } return mean } ================================================ FILE: src/backend/main/go.main/helpers/discussion_test.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package helpers import ( "testing" ) func TestComputeDiscussionIL(t *testing.T) { sets := [][]float64{ {1, 2, 2, 2, 5, -1, 10, 5, 5, 6}, {-1, -2, 3, 2, 5, -1, 10, 5, 5, 6}, {1, 2}, {5}, {-10}, {9, -10}, {-1, -1}, } for i, set := range sets { IL := ComputeDiscussionIL(set) if IL == 0 { t.Errorf("got 0 result for set %d", i) } } } ================================================ FILE: src/backend/main/go.main/helpers/filesystem.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package helpers /* import ( "github.com/spf13/afero" "os" ) func init() { var AppFs afero.Fs = afero.NewOsFs() } func Exists(path string) (bool, error) { _, err := afero.Fs.Stat(path) if err == nil { return true, nil } if os.IsNotExist(err) { return false, nil } return false, err } */ ================================================ FILE: src/backend/main/go.main/helpers/misc.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package helpers func EscapeUsername(username string) string { // TODO : implement an algorithm against injections return username } ================================================ FILE: src/backend/main/go.main/helpers/netTest.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package helpers import ( "crypto/rsa" "crypto/tls" "encoding/hex" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "math/big" "net" "time" ) // netTest provides mocked network structs and interfaces to run tests which need networking var ( tlsConfig *tls.Config testRSACertificate = fromHex("3082024b308201b4a003020102020900e8f09d3fe25beaa6300d06092a864886f70d01010b0500301f310b3009060355040a1302476f3110300e06035504031307476f20526f6f74301e170d3136303130313030303030305a170d3235303130313030303030305a301a310b3009060355040a1302476f310b300906035504031302476f30819f300d06092a864886f70d010101050003818d0030818902818100db467d932e12270648bc062821ab7ec4b6a25dfe1e5245887a3647a5080d92425bc281c0be97799840fb4f6d14fd2b138bc2a52e67d8d4099ed62238b74a0b74732bc234f1d193e596d9747bf3589f6c613cc0b041d4d92b2b2423775b1c3bbd755dce2054cfa163871d1e24c4f31d1a508baab61443ed97a77562f414c852d70203010001a38193308190300e0603551d0f0101ff0404030205a0301d0603551d250416301406082b0601050507030106082b06010505070302300c0603551d130101ff0402300030190603551d0e041204109f91161f43433e49a6de6db680d79f60301b0603551d230414301280104813494d137e1631bba301d5acab6e7b30190603551d1104123010820e6578616d706c652e676f6c616e67300d06092a864886f70d01010b0500038181009d30cc402b5b50a061cbbae55358e1ed8328a9581aa938a495a1ac315a1a84663d43d32dd90bf297dfd320643892243a00bccf9c7db74020015faad3166109a276fd13c3cce10c5ceeb18782f16c04ed73bbb343778d0c1cf10fa1d8408361c94c722b9daedb4606064df4c1b33ec0d1bd42d4dbfe3d1360845c21d33be9fae7") testRSAPrivateKey = &rsa.PrivateKey{ PublicKey: rsa.PublicKey{ N: bigFromString("153980389784927331788354528594524332344709972855165340650588877572729725338415474372475094155672066328274535240275856844648695200875763869073572078279316458648124537905600131008790701752441155668003033945258023841165089852359980273279085783159654751552359397986180318708491098942831252291841441726305535546071"), E: 65537, }, D: bigFromString("7746362285745539358014631136245887418412633787074173796862711588221766398229333338511838891484974940633857861775630560092874987828057333663969469797013996401149696897591265769095952887917296740109742927689053276850469671231961384712725169432413343763989564437170644270643461665184965150423819594083121075825"), Primes: []*big.Int{ bigFromString("13299275414352936908236095374926261633419699590839189494995965049151460173257838079863316944311313904000258169883815802963543635820059341150014695560313417"), bigFromString("11578103692682951732111718237224894755352163854919244905974423810539077224889290605729035287537520656160688625383765857517518932447378594964220731750802463"), }, } ) func init() { tlsConfig = &tls.Config{ Time: func() time.Time { return time.Unix(0, 0) }, Rand: zeroSource{}, Certificates: make([]tls.Certificate, 1), InsecureSkipVerify: true, MinVersion: tls.VersionSSL30, MaxVersion: tls.VersionTLS12, CipherSuites: allCipherSuites(), } tlsConfig.Certificates[0].Certificate = [][]byte{testRSACertificate} tlsConfig.Certificates[0].PrivateKey = testRSAPrivateKey tlsConfig.BuildNameToCertificate() } // GetTlsConn returns a tls Conn suitable fo tests func GetTestTlsConn() *tls.Conn { _, serverConn := net.Pipe() return tls.Server(serverConn, tlsConfig.Clone()) } // zeroSource is an io.Reader that returns an unlimited number of zero bytes. type zeroSource struct{} func (zeroSource) Read(b []byte) (n int, err error) { for i := range b { b[i] = 0 } return len(b), nil } func fromHex(s string) []byte { b, _ := hex.DecodeString(s) return b } func bigFromString(s string) *big.Int { ret := new(big.Int) ret.SetString(s, 10) return ret } func allCipherSuites() []uint16 { ids := make([]uint16, len(TlsSuites)) var i int for id := range TlsSuites { ids[i] = id i++ } return ids } ================================================ FILE: src/backend/main/go.main/helpers/patch.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package helpers import ( "encoding/json" "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/tidwall/gjson" "gopkg.in/oleiade/reflections.v1" "reflect" ) type errHandler struct { err error } type patch struct { actor Initiator currentState ObjectPatchable dbState ObjectPatchable fieldsInCurrentState map[string]interface{} fieldsInPatch map[string]interface{} jsonMap map[string]string // keys are json properties, values are struct Field name counterpart newState ObjectPatchable raw []byte } // UpdateWithPatch replace obj with updated version according to patch directives if everything went well. // It returns fields that have been effectively modified. func UpdateWithPatch(patch []byte, obj ObjectPatchable, actor Initiator) (newObj ObjectPatchable, modifiedFields map[string]interface{}, err error) { p, err := buildPatch(patch, obj, actor) if err != nil { return obj, nil, err } err = validateCurrentState(p) if err != nil { return obj, nil, err } // squash newState values with dbState values for those that it's not allowed to modify validatedFields, err := validateActorRights(p.dbState, p.newState, p.actor) if err != nil { return obj, nil, err } // ensure mandatory properties are present p.newState.MarshallNew() // copy fields that have been validated and modified into the map that will be returned. modifiedFields = map[string]interface{}{} for key := range p.fieldsInPatch { fieldName := p.jsonMap[key] if value, ok := validatedFields[fieldName]; ok { modifiedFields[fieldName] = value } } return p.newState, modifiedFields, nil } // func buildPatch(rawPatch []byte, dbState ObjectPatchable, actor Initiator) (*patch, error) { patch := patch{ actor: actor, dbState: dbState, fieldsInCurrentState: map[string]interface{}{}, fieldsInPatch: map[string]interface{}{}, jsonMap: dbState.JsonTags(), raw: rawPatch, } // unmarshal raw json to a map[string]interface{} err := json.Unmarshal(rawPatch, &patch.fieldsInPatch) if err != nil { return nil, err } // get raw 'current_state' from patch and apply it to currentState object patch.currentState = dbState.NewEmpty().(ObjectPatchable) reflect.ValueOf(patch.currentState).Elem().Set(reflect.ValueOf(dbState).Elem()) // now patch.currentState has same kind than object counterpart patch.fieldsInCurrentState = patch.fieldsInPatch["current_state"].(map[string]interface{}) err = patch.currentState.UnmarshalMap(patch.fieldsInCurrentState) if err != nil { return nil, err } // remove 'current_state' before creating the newState from remaining properties delete(patch.fieldsInPatch, "current_state") patch.newState = dbState.NewEmpty().(ObjectPatchable) reflect.ValueOf(patch.newState).Elem().Set(reflect.ValueOf(dbState).Elem()) err = patch.newState.UnmarshalMap(patch.fieldsInPatch) if err != nil { return nil, err } return &patch, nil } // func validateCurrentState(patch *patch) (err error) { // before comparing dbState and currentState we must ensure that embedded slices are sorted the same way patch.dbState.SortSlices() patch.currentState.SortSlices() patch.newState.SortSlices() // check global equality // TODO: add a compare(a,b T) func to CaliopenObject interface if !reflect.DeepEqual(patch.dbState, patch.currentState) { return NewCaliopenErr(UnprocessableCaliopenErr, "current state not consistent with db state") } // seek for keys within patch that are not in current_state meaning patch wants to add the key, // consequently the value in db should equals to default zero emptyState := patch.dbState.NewEmpty() for key := range patch.fieldsInPatch { field := patch.jsonMap[key] if ok, err := reflections.HasField(patch.dbState, field); err != nil || !ok { return NewCaliopenErrf(UnprocessableCaliopenErr, "struct %s has no key %s", reflect.TypeOf(patch.dbState).String(), key) } if _, present := patch.fieldsInCurrentState[key]; !present { empty, err1 := reflections.GetField(emptyState, field) store, err2 := reflections.GetField(patch.dbState, field) if err1 != nil || err2 != nil { return NewCaliopenErrf(UnprocessableCaliopenErr, "[Patch] failed to retrieve field <%s> from object", field) } if !reflect.DeepEqual(store, empty) { return NewCaliopenErrf(UnprocessableCaliopenErr, "[Patch] field <%s> not consistent with db state", field) } } } return nil } // validateActorRights iterate recursively over dbState fields to prevent actor to modify those that he hasn't right on. // If a field is protected against current actor, its value is replaced by its value from db, // otherwise field name and value are copied to validatedFields for later use // dbState and newState must be pointers to structs, but dbState could be a nil pointer func validateActorRights(dbState, newState interface{}, actor Initiator) (validatedFields map[string]interface{}, err error) { validatedFields = map[string]interface{}{} fieldsCount := reflect.TypeOf(newState).Elem().NumField() for i := 0; i < fieldsCount; i++ { fieldName := reflect.TypeOf(newState).Elem().Field(i).Name if canModifyProperty(newState, fieldName, actor) { fieldKind, _ := reflections.GetFieldKind(newState, fieldName) switch fieldKind { case reflect.Slice: sliceNew := reflect.ValueOf(newState).Elem().FieldByName(fieldName) // check if it's a slice of structs if sliceNew.Len() > 0 { switch sliceNew.Index(0).Kind() { case reflect.Struct: if _, ok := sliceNew.Index(0).Addr().Interface().(ObjectPatchable); ok { // iterate over items to check rights on sub-fields. sliceDB := reflect.ValueOf(dbState).Elem().FieldByName(fieldName) for i := 0; i < sliceNew.Len(); i++ { // check that we have a db counterpart before calling validateActorRights if i < sliceDB.Len() { _, err := validateActorRights(sliceDB.Index(i).Addr().Interface(), sliceNew.Index(i).Addr().Interface(), actor) if err != nil { return nil, err } } else { _, err := validateActorRights(nil, sliceNew.Index(i).Addr().Interface(), actor) if err != nil { return nil, err } } } } } } validatedFields[fieldName], _ = reflections.GetField(newState, fieldName) case reflect.Struct: subNew, _ := reflections.GetField(newState, fieldName) if _, ok := subNew.(ObjectPatchable); ok { // we must check rights on sub-fields. var subDB *interface{} if dbState != nil { f, err := reflections.GetField(dbState, fieldName) if err != nil { return nil, WrapCaliopenErr(err, UnprocessableCaliopenErr, "validateActorRights failed") } subDB = &f } else { subDB = nil } _, err = validateActorRights(subDB, &subNew, actor) if err != nil { return nil, err } validatedFields[fieldName], _ = reflections.GetField(newState, fieldName) } default: // no sub-level, field could be updated directly validatedFields[fieldName], _ = reflections.GetField(newState, fieldName) } } else { // actor can't patch this field if dbState != nil { // try to silently discard the field by replacing it with value from db dbValue, err := reflections.GetField(dbState, fieldName) if err != nil { return nil, WrapCaliopenErrf(err, UnknownCaliopenErr, "validateActorRights failed to fetch field %s", fieldName) } reflections.SetField(newState, fieldName, dbValue) } else { // no counterpart available in db, set field to zero fieldType := reflect.TypeOf(newState).Elem().Field(i).Type reflections.SetField(newState, fieldName, reflect.New(fieldType).Elem().Interface()) } } } return validatedFields, nil } // canModifyProperty returns true if and only if Initiator has rights to modify the property of the object. // directive comes from the "patch" tag found within struct declaration func canModifyProperty(obj interface{}, field string, actor Initiator) bool { rightLevel := Unknown directive, err := reflections.GetFieldTag(obj, field, "patch") if directive != "" && err == nil { if level, ok := Initiators[directive]; ok { rightLevel = level } } return rightLevel >= actor // if rightLevel is below actor level, then actor has not the right to modify } // func to do the parsing once and get a pointer to the result. func ParsePatch(json []byte) (*gjson.Result, error) { if !gjson.Valid(string(json)) { return nil, errors.New("invalid json") } r := gjson.ParseBytes(json) return &r, nil } ================================================ FILE: src/backend/main/go.main/helpers/uuid.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. * * // const, GetTime & ToUnixTime borrowed from github.com/google/uuid */ package helpers import ( "encoding/binary" "github.com/satori/go.uuid" "time" ) const ( lillian = 2299160 // Julian day of 15 Oct 1582 unix = 2440587 // Julian day of 1 Jan 1970 epoch = unix - lillian // Days between epochs g1582 = epoch * 86400 // seconds between epochs g1582ns100 = g1582 * 10000000 // 100s of a nanoseconds between epochs ) // GetTime returns the time in 100s of nanoseconds since 15 Oct 1582 encoded in // uuid. The time is only defined for version 1 and 2 UUIDs. func GetTime(uuid uuid.UUID) time.Time { t := int64(binary.BigEndian.Uint32(uuid[0:4])) t |= int64(binary.BigEndian.Uint16(uuid[4:6])) << 32 t |= int64(binary.BigEndian.Uint16(uuid[6:8])&0xfff) << 48 return time.Unix(toUnixTime(t)) } // ToUnixTime converts t the number of seconds and nanoseconds using the Unix // epoch of 1 Jan 1970. func toUnixTime(t int64) (sec, nsec int64) { sec = int64(t - g1582ns100) nsec = (sec % 10000000) * 100 sec /= 10000000 return sec, nsec } ================================================ FILE: src/backend/main/go.main/messages/messages.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package messages import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/microcosm-cc/bluemonday" "golang.org/x/net/html" "regexp" "strings" "unicode" "unicode/utf8" ) // scrub message's bodies to make message displayable in frontend interfaces. // message is modified in-place. // if sanitation failed, message's string bodies are emptied func SanitizeMessageBodies(msg *Message) { p := CaliopenPolicy() (*msg).Body_html = p.Sanitize(msg.Body_html) (*msg).Body_html = replaceBodyTag(msg.Body_html) } // UGCPolicy returns a policy aimed at user generated content that is a result // of HTML WYSIWYG tools and Markdown conversions. // // This is expected to be a fairly rich document where as much markup as // possible should be retained. Markdown permits raw HTML so we are basically // providing a policy to sanitise HTML5 documents safely but with the // least intrusion on the formatting expectations of the user. // //See https://github.com/microcosm-cc/bluemonday to build a bespoke policy. func CaliopenPolicy() *bluemonday.Policy { basePolicy := bluemonday.UGCPolicy() basePolicy.AllowAttrs("title").Matching(regexp.MustCompile(`[\p{L}\p{N}\s\-_',:\[\]!\./\\\(\)&]*`)).Globally() basePolicy.RequireNoFollowOnFullyQualifiedLinks(true) basePolicy.AddTargetBlankToFullyQualifiedLinks(true) // allow body with few attributes basePolicy.AllowElements("body") basePolicy.AllowAttrs("leftmargin").Matching(bluemonday.Integer).OnElements("body") basePolicy.AllowAttrs("rightmargin").Matching(bluemonday.Integer).OnElements("body") basePolicy.AllowAttrs("topmargin").Matching(bluemonday.Integer).OnElements("body") basePolicy.AllowAttrs("bottommargin").Matching(bluemonday.Integer).OnElements("body") basePolicy.AllowAttrs("marginwidth").Matching(bluemonday.Integer).OnElements("body") basePolicy.AllowAttrs("marginheight").Matching(bluemonday.Integer).OnElements("body") basePolicy.AllowAttrs("offset").Matching(bluemonday.Integer).OnElements("body") // allow img src="data:… basePolicy.AllowDataURIImages() return basePolicy } // Returns an excerpt of Message from either body_plain, body_html or subject. // Excerpt is always a plain text without any markup, of a length of 'length' runes max. // Best effort is made to retrieve relevant excerpt from html body (see excerptFromHMTL func) // A string is always returned, even if excerpt extraction failed. // If option "wordWrap" is true, string is trimmed at the end of a word, thus it may be shorter than length. // If option "addEllipsis" is true, … (unicode 2026) is added at the end of the string if the string has been shortened. func ExcerptMessage(msg Message, length int, wordWrap, addEllipsis bool) (excerpt string) { // 1. try to extract excerpt from HTML if msg.Body_html != "" { var err error excerpt, err = excerptFromHMTL(msg.Body_html) if err == nil && excerpt != "" { return trimExcerpt(excerpt, length, wordWrap, addEllipsis) } } // 2. fall-back to plain body if any if msg.Body_plain != "" { return trimExcerpt(msg.Body_plain, length, wordWrap, addEllipsis) } // 3. then subject if msg.Subject != "" { return trimExcerpt(msg.Subject, length, wordWrap, addEllipsis) } // 4. nothing found, return empty string return "" } // algorithm to retrieve the more relevant excerpt from an HMTL doc. // still WIP in september 2017 func excerptFromHMTL(source string) (excerpt string, err error) { p := CaliopenPolicy() sanitized := p.Sanitize(source) sanitized = replaceBodyTag(sanitized) doc, err := html.Parse(strings.NewReader(sanitized)) if err != nil { return "", err } excerpt_strings := []string{} var f func(*html.Node) f = func(n *html.Node) { //take textNode only, which are not within anchor node if n.Type == html.TextNode && n.Data != "" && n.Parent.Data != "a" { //remove lines filled with only spaces and/or control chars trim_str := strings.TrimFunc(n.Data, func(r rune) bool { return unicode.IsControl(r) || unicode.IsSpace(r) }) if len(trim_str) > 0 { excerpt_strings = append(excerpt_strings, trim_str) } } for c := n.FirstChild; c != nil; c = c.NextSibling { f(c) } } f(doc) excerpt = strings.Join(excerpt_strings, " ") excerpt = html.UnescapeString(excerpt) return } func trimExcerpt(s string, l int, wordWrap, addEllipsis bool) string { trimmed := truncate_unicode(s, l) if wordWrap { last_valid, width := lastIndexPunctuation(trimmed) if last_valid != len(trimmed) && last_valid != -1 { //punctuation found, but not at end of string trimmed = trimmed[:last_valid-width] } } if addEllipsis { if len(trimmed) < len(s) { return trimmed + "…" } } return trimmed } func truncate_unicode(s string, l int) string { if l == 0 { return "" } runesCount := utf8.RuneCountInString(s) if runesCount <= l { return s } runesString := []rune(s) return string(runesString[:l]) } func lastIndexPunctuation(s string) (index, width int) { for i, w := len(s), 0; i > 0; i -= w { runeValue, width := utf8.DecodeLastRuneInString(s[:i]) if unicode.IsPunct(runeValue) || unicode.IsSpace(runeValue) { return i, width } w = width } return -1, 0 } func replaceBodyTag(in string) (out string) { bodyStart := strings.NewReplacer(``, `/div>`) out = bodyEnd.Replace(out) return out } ================================================ FILE: src/backend/main/go.main/pi/identities.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package pi import ( "context" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "time" ) func UpdatePIContactIdentity(ctx context.Context, contact *Contact, identity *ContactIdentity) { // TODO: real update. But how to do it ??? // ctx should be use to pass context information to this calculator // few computations below are just for demonstration if contact.PrivacyIndex != nil { identity.PrivacyIndex.Comportment = contact.PrivacyIndex.Comportment identity.PrivacyIndex.Context = contact.PrivacyIndex.Context identity.PrivacyIndex.Technic = contact.PrivacyIndex.Technic } else { identity.PrivacyIndex = PrivacyIndex{} } switch identity.Protocol { case "email": identity.PrivacyIndex.Technic-- case "telephone": identity.PrivacyIndex.Technic++ case "im": identity.PrivacyIndex.Technic -= 2 } identity.PrivacyIndex.DateUpdate = time.Now() } ================================================ FILE: src/backend/main/go.main/pi/message.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package pi import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "math" "strconv" "strings" ) // ComputePIMessage returns estimated PIMessage values func ComputePIMessage(message *Message) *PIMessage { piMessage := &PIMessage{Transport: 0, Social: 0, Content: 0} features := *message.Privacy_features // pi.Content if encrypted, ok := features["messsage_encryption_method"]; ok && encrypted != "" { piMessage.Content += 50 } signed, ok := features["message_signed"] // TODO : have a correct unmarshalling of typed features if ok && len(signed) > 0 && strings.ToLower(string(signed[0])) == "t" { piMessage.Content += 20 } // pi.Social known_participants := 0.0 for _, participant := range message.Participants { if len(participant.Contact_ids) > 0 { known_participants += 1 } } if known_participants > 1 { // user contact is included known_participants -= 1 } ratio := known_participants / float64(len(message.Participants)) * 100 piMessage.Social += uint32(math.Min(80, ratio)) // pi.Transport trSigned, ok := features["transport_signed"] // TODO : have a correct unmarshalling of typed features if ok && len(trSigned) > 0 && strings.ToLower(string(trSigned[0])) == "t" { piMessage.Transport += 20 } nbHops, ok := features["nb_external_hops"] if ok { value, err := strconv.Atoi(nbHops) if err == nil && value <= 5 { bump := float64((-5 * value) + 25.0) piMessage.Transport += uint32(math.Max(0, bump)) } } inCipher, ok := features["ingress_cipher"] if ok && inCipher != "" { piMessage.Transport += 20 } // TODO : normalize return piMessage } ================================================ FILE: src/backend/main/go.main/users/oauth2.go ================================================ package users import ( "context" "crypto/rand" "encoding/base64" "errors" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" "github.com/Sirupsen/logrus" "golang.org/x/oauth2" googleOAuth2 "golang.org/x/oauth2/google" "time" ) const ( CALLBACK_BASE_URI = "/api/v2/providers/%s/callback" CRED_ACCESS_TOKEN = "oauth2accesstoken" CRED_REFRESH_TOKEN = "oauh2refreshtoken" CRED_TOKEN_TYPE = "tokentype" CRED_TOKEN_EXPIRY = "tokenexpiry" CRED_USERNAME = "username" ) type Oauth2Interfacer interface { GetProviders() map[string]Provider GetHostname() string GetIdentityStore() backends.IdentityStorageUpdater } // ValidateOauth2Credentials wraps methods to check Oauth2 access token validity for different providers. // If oauth2 access token has expired it's renewed using oauth2 refresh token. // New access token is embedded in userIdentity, and stored in db if updateStore is set to true. func ValidateOauth2Credentials(userIdentity *UserIdentity, interfacer Oauth2Interfacer, updateStore bool) CaliopenError { switch userIdentity.Infos["provider"] { case "gmail": gmail, gotProvider := interfacer.GetProviders()["gmail"] if !gotProvider { return NewCaliopenErr(FailDependencyCaliopenErr, "failed to find gmail provider params in providers map") } credentialsUpdated, err := getValidGmailAccessToken(userIdentity, gmail, interfacer.GetHostname()) if err != nil { return WrapCaliopenErr(err, WrongCredentialsErr, "failed to get valid gmail access token") } if credentialsUpdated && updateStore { credentials := userIdentity.Credentials store := interfacer.GetIdentityStore() err := store.UpdateUserIdentity(userIdentity, map[string]interface{}{ "Credentials": userIdentity.Credentials, }) // re-embed credentials in userIdentity because store.UpdateUserIdentity has removed it from UserIdentity struct userIdentity.Credentials = credentials if err != nil { return WrapCaliopenErr(err, WrongCredentialsErr, "imapLogin failed to update access token in store") } } default: return NewCaliopenErrf(FailDependencyCaliopenErr, "unhandled oauth2 provider <%v>", interfacer.GetProviders()[userIdentity.Infos["provider"]]) } return nil } /* Google services */ func SetGoogleAuthRequestUrl(provider *Provider, hostname string) (state string, err error) { provider.OauthCallbackUri = fmt.Sprintf(CALLBACK_BASE_URI, "gmail") config := SetGoogleOauthConfig(*provider, hostname) state = randomState() provider.OauthRequestUrl = config.AuthCodeURL(state, oauth2.AccessTypeOffline) return } func SetGoogleOauthConfig(provider Provider, hostname string) *oauth2.Config { return &oauth2.Config{ ClientID: provider.Infos["client_id"], ClientSecret: provider.Infos["client_secret"], Endpoint: googleOAuth2.Endpoint, RedirectURL: hostname + provider.OauthCallbackUri, Scopes: []string{"profile", "email", "https://mail.google.com/"}, } } func SetMastodonOauthConfig(provider Provider, hostname string) *oauth2.Config { return &oauth2.Config{ ClientID: provider.Infos["client_id"], ClientSecret: provider.Infos["client_secret"], Endpoint: oauth2.Endpoint{ AuthURL: provider.Infos["address"] + "/oauth/authorize", TokenURL: provider.Infos["address"] + "/oauth/token", AuthStyle: oauth2.AuthStyleInParams, }, RedirectURL: hostname + provider.OauthCallbackUri, Scopes: []string{"read", "write", "follow"}, } } // GetValidGmailAccessToken checks identity's access token validity. // If token has expired a new one is retrieved by the mean of refresh token, // new credentials are embedded in user identity and credentialsUpdated is set to true. // UserIdentity MUST carry identity's credentials. It's caller responsibility to store new credentials in db. func getValidGmailAccessToken(uId *UserIdentity, provider Provider, hostname string) (credentialsUpdated bool, err error) { if uId.Credentials == nil { err = errors.New("[GetValidGmailAccessToken] missing credentials in user identity") return } expiry, err := time.Parse(time.RFC3339, (*uId.Credentials)[CRED_TOKEN_EXPIRY]) if err != nil { logrus.Error(err) } restoredToken := &oauth2.Token{ AccessToken: (*uId.Credentials)[CRED_ACCESS_TOKEN], TokenType: (*uId.Credentials)[CRED_TOKEN_TYPE], RefreshToken: (*uId.Credentials)[CRED_REFRESH_TOKEN], Expiry: expiry, } //logrus.Infof("restoredToken : %+v\n\n", restoredToken) if restoredToken.Expiry.IsZero() || !restoredToken.Valid() { // need a new token oauthConfig := SetGoogleOauthConfig(provider, hostname) ctx := context.TODO() //logrus.Infof("oauthConfig : %+v\n\n", oauthConfig) tokenSource := oauthConfig.TokenSource(ctx, restoredToken) updatedToken, tokenErr := tokenSource.Token() if tokenErr != nil { logrus.Errorf("[getValidGmailAccessToken]TokenSource error : %+v", tokenErr) err = tokenErr return } (*uId.Credentials)[CRED_ACCESS_TOKEN] = updatedToken.AccessToken (*uId.Credentials)[CRED_REFRESH_TOKEN] = updatedToken.RefreshToken (*uId.Credentials)[CRED_TOKEN_EXPIRY] = updatedToken.Expiry.Format(time.RFC3339) (*uId.Credentials)[CRED_TOKEN_TYPE] = updatedToken.TokenType credentialsUpdated = true return } return } /* Mastodon API */ func SetMastodonAuthRequestUrl(provider *Provider, hostname string) (state string, err error) { state = randomState() provider.OauthCallbackUri = fmt.Sprintf(CALLBACK_BASE_URI, "mastodon") config := SetMastodonOauthConfig(*provider, hostname) provider.OauthRequestUrl = config.AuthCodeURL(state, oauth2.AccessTypeOffline) return } // Returns a base64 encoded random 32 byte string. func randomState() string { b := make([]byte, 32) rand.Read(b) return base64.RawURLEncoding.EncodeToString(b) } /* end of Google services*/ ================================================ FILE: src/backend/main/go.main/users/password.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package users import ( "errors" "strconv" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" "github.com/nbutton23/zxcvbn-go" "github.com/tidwall/gjson" "golang.org/x/crypto/bcrypt" ) const ( defaultBcryptCost = 12 // 12 is the default cost of python's bcrypt lib passwordStrengthkey = "password_strength" // key in privacy_features map ) func ChangeUserPassword(user *User, patch *gjson.Result, store backends.UserStorage) error { // verify that current_password in patch is the good one current_pwd := patch.Get("current_state.password").Str err := bcrypt.CompareHashAndPassword(user.Password, []byte(current_pwd)) if err != nil { return errors.New("old password is incorrect") } new_password := patch.Get("password").Str return ResetUserPassword(user, new_password, store) } func ResetUserPassword(user *User, new_password string, store backends.UserStorage) error { //compute new password strength user_infos := []string{user.Name, user.GivenName, user.FamilyName, user.RecoveryEmail} user_infos = append(user_infos, user.LocalIdentities...) scoring := zxcvbn.PasswordStrength(new_password, user_infos) (*user.PrivacyFeatures)[passwordStrengthkey] = strconv.FormatInt(int64(scoring.Score), 10) // hash new password and store it hashpass, err := bcrypt.GenerateFromPassword([]byte(new_password), defaultBcryptCost) (*user).Password = hashpass err = store.UpdateUserPasswordHash(user) if err != nil { return errors.New("[ChangeUserPassword] failed to store updated user : " + err.Error()) } return nil } ================================================ FILE: src/backend/main/py.main/CHANGES.rst ================================================ 0.0.1 ----- - Initial version 0.0.2 ----- - Code refactoring into a single repository. Caliopen platform is made of 3 python packages : caliopen.main, caliopen_storage. and caliopen.server (ie the REST HTTP API) ================================================ FILE: src/backend/main/py.main/MANIFEST.in ================================================ include *.cfg *.rst *.template ================================================ FILE: src/backend/main/py.main/README.rst ================================================ Entry point =========== This repository is part of CaliOpen platform. For documentation, installation and contribution instructions, please refer to https://caliopen.github.io Caliopen Main package ============= This is the main entry point for whole application. ================================================ FILE: src/backend/main/py.main/caliopen_main/__init__.py ================================================ # -*- coding: utf-8 -*- __version__ = '0.23.0' ================================================ FILE: src/backend/main/py.main/caliopen_main/common/__init__.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/common/core/__init__.py ================================================ # -*- coding: utf-8 -*- """Caliopen common core classes.""" from __future__ import absolute_import, print_function, unicode_literals from .base import BaseUserCore from .pubkey import PublicKey from .related import BaseUserRelatedCore __all__ = ['BaseUserCore', 'PublicKey', 'BaseUserRelatedCore'] ================================================ FILE: src/backend/main/py.main/caliopen_main/common/core/base.py ================================================ """Caliopen core base class for objects belong to an user.""" from caliopen_storage.core import BaseCore from caliopen_storage.exception import NotFound class BaseUserCore(BaseCore): """Used by objects related to only one user (most core).""" _user = None @property def user(self): """Return user related to this object.""" from caliopen_main.user.core import User if not self._user: self._user = User.get(self.user_id) return self._user @classmethod def get(cls, user, obj_id): """Get a core object belong to user, with model related id.""" param = {cls._pkey_name: obj_id} obj = cls._model_class.get(user_id=user.user_id, **param) if obj: return cls(obj) raise NotFound('%s #%s not found for user %s' % (cls.__class__.name, obj_id, user.user_id)) @classmethod def get_by_user_id(cls, user_id, obj_id): """Get a core object belong to user, with model related id.""" param = {cls._pkey_name: obj_id} obj = cls._model_class.get(user_id=user_id, **param) if obj: return cls(obj) raise NotFound('%s #%s not found for user %s' % (cls.__class__.name, obj_id, user_id)) @classmethod def find(cls, user, filters=None, limit=None, offset=0, count=False): """ Find core objects that belong to an user. can only use columns part of primary key """ q = cls._model_class.filter(user_id=user.user_id) if not filters: objs = q else: objs = q.filter(**filters) if count: return objs.count() if limit or offset: objs = objs[offset:(limit + offset)] return {'objects': [cls(x) for x in objs], 'total': len(q)} @classmethod def count(cls, user, filters=None): """Count core objects that belong to an user.""" return cls.find(user, filters, count=True) @classmethod def create(cls, user, **attrs): """Create a core object belong to an user.""" obj = cls._model_class.create(user_id=user.user_id, **attrs) return cls(obj) @classmethod def belongs_to_user(cls, user_id, object_id): """Test if an object belong to an user.""" param = {cls._pkey_name: object_id} obj = cls._model_class.get(user_id=user_id, **param) if obj: return True return False ================================================ FILE: src/backend/main/py.main/caliopen_main/common/core/pubkey.py ================================================ # -*- coding: utf-8 -*- """Caliopen public key core classes.""" from __future__ import absolute_import, print_function, unicode_literals import uuid from caliopen_main.common.store import PublicKey as ModelPublicKey from caliopen_main.common.core.related import BaseUserRelatedCore class PublicKey(BaseUserRelatedCore): """Public key core class.""" _model_class = ModelPublicKey _pkey_name = 'key_id' @classmethod def find(cls, user, resource_id): """Get public keys for an user and a resource.""" models = cls._model_class.filter(user_id=user.user_id, resource_id=resource_id) for m in models: yield cls(m) @classmethod def create(cls, user, resource_id, resource_type, **kwargs): """Create a new public key related to an user and a resource.""" if 'key_id' not in kwargs: kwargs['key_id'] = uuid.uuid4() obj = cls._model_class.create(user_id=user.user_id, resource_id=resource_id, resource_type=resource_type, **kwargs) return cls(obj) ================================================ FILE: src/backend/main/py.main/caliopen_main/common/core/related.py ================================================ # -*- coding: utf-8 -*- """Caliopen core related classes.""" from __future__ import absolute_import, print_function, unicode_literals import logging from caliopen_storage.core import BaseCore log = logging.getLogger(__name__) class BaseUserRelatedCore(BaseCore): """Core class for related objects to an user and another entity.""" _pkey_name = None # to be defined in real class @classmethod def create(cls, user, resource_id, **kwargs): """Create a related user and resource entity.""" obj = cls._model_class.create(user_id=user.user_id, resource_id=resource_id, **kwargs) return cls(obj) @classmethod def get(cls, user, resource_id, value): """Get a related entity.""" kwargs = {'user_id': user.user_id, 'resource_id': resource_id, cls._pkey_name: value} try: obj = cls._model_class.get(**kwargs) return cls(obj) except Exception as exc: log.exception('Unexpected error during retrieve of resource %s' % exc) return None @classmethod def find(cls, user, resource_id, filters=None): """Find related object for an user and an given resource.""" filters = filters if filters else {} filters.update({'user_id': user.user_id, 'resource_id': resource_id}) q = cls._model_class.filter(**filters) if not filters: objs = q else: objs = q.filter(**filters) return {'total': len(objs), 'data': [cls(x) for x in objs]} def to_dict(self): """Return a dict representation.""" return {col: getattr(self, col) for col in self._model_class._columns.keys()} ================================================ FILE: src/backend/main/py.main/caliopen_main/common/errors.py ================================================ class PatchUnprocessable(Exception): """Exception when patch dict is malformed or unprocessable.""" def __init__(self, message=None, **kw): Exception.__init__(self, message, **kw) class PatchError(Exception): """Exception when processing patch was unsuccessfull""" def __init__(self, message=None, **kw): Exception.__init__(self, message, **kw) class PatchConflict(Exception): """Exception when processing patch was unsuccessfull""" def __init__(self, message=None, **kw): Exception.__init__(self, message, **kw) class ObjectInitFailed(Exception): """Exception when __init__ func failed to process object initialization""" def __init__(self, message=None, **kw): Exception.__init__(self, message, **kw) class ForbiddenAction(Exception): """ Exception when an user tries to do something forbidden because of insufficient rights or because action is not allowed on a specific object or attribute """ def __init__(self, message=None, **kw): Exception.__init__(self, message, **kw) class DuplicateMessage(Exception): """Exception when processing ingress messages already imported for user""" def __init__(self, message=None, **kw): Exception.__init__(self, message, **kw) ================================================ FILE: src/backend/main/py.main/caliopen_main/common/helpers/__init__.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/common/helpers/normalize.py ================================================ # -*- coding: utf-8 -*- """Normalization functions for different values.""" from __future__ import absolute_import, unicode_literals import re import logging from email.utils import parseaddr log = logging.getLogger(__name__) mastodon_url_regex = '^https:\/\/(.*)\/@(.*)' mastodon_url_legacy_regex = '^https:\/\/(.*)\/users\/(.*)' def clean_email_address(addr): """Clean an email address for user resolve.""" try: real_name, email = parseaddr(addr.replace('\r', '')) except UnicodeError: addr = addr.decode('utf-8', errors='ignore') real_name, email = parseaddr(addr.replace('\r', '')) err_msg = 'Invalid email address {}'.format(addr) if not email or '@' not in email: # Try something else log.info('Last chance email parsing for {}'.format(addr)) matches = re.match('(.*)<(.*@.*)>', addr) if matches and matches.groups(): real_name, email = matches.groups() else: log.warn(err_msg) return ("", "") name, domain = email.lower().split('@', 1) if '@' in domain: log.error(err_msg) return ("", "") if '+' in name: try: name, ext = name.split('+', 1) except Exception as exc: log.info(exc) # unicode everywhere return (u'%s@%s' % (name, domain), email) def clean_twitter_address(addr): return addr.strip('@').lower() def clean_mastodon_address(addr): return addr.strip('@').lower().split('@') def parse_mastodon_url(url): """extract username and domain from a mastodon account url in the format https://instance.tld/@username :return: tuple (server, username) """ matches = re.findall(mastodon_url_regex, url) if len(matches) != 1 or len(matches[0]) != 2: # try legacy fallback matches = re.findall(mastodon_url_legacy_regex, url) if len(matches) != 1 or len(matches[0]) != 2: raise SyntaxError return matches[0] ================================================ FILE: src/backend/main/py.main/caliopen_main/common/helpers/strings.py ================================================ # -*- coding: utf-8 -*- """helpers to work with strings""" from __future__ import absolute_import, unicode_literals import re import logging log = logging.getLogger(__name__) def unicode_truncate(s, length): """"Truncate string after `length` bytes less trailing unicode char.""" if isinstance(s, str): s = s.decode("utf8", 'ignore') partial = s[:length] return re.sub("([\xf6-\xf7][\x80-\xbf]{0,2}|[\xe0-\xef][\x80-\xbf]{0,1}" "|[\xc0-\xdf])$", "", partial) def to_utf8(input, charset): """Convert input string to utf-8 return input string if it fails. :param input: string :param charset: string :return: utf-8 string """ if charset: matches = re.match('^charset.*"(.*)"', charset) if matches and matches.groups(): charset = matches.groups()[0] try: return input.decode(charset, "replace"). \ encode("utf-8", "replace") except UnicodeError as exc: log.warn("decoding <{}> string to utf-8 failed " "with error : {}".format(input, exc)) return input else: try: return input.decode("us-ascii", "replace"). \ encode("utf-8", "replace") except Exception as exc: log.warn("decoding <{}> string to utf-8 failed " "with error : {}".format(bytes(input), exc)) return input ================================================ FILE: src/backend/main/py.main/caliopen_main/common/interfaces/IO.py ================================================ # -*- coding: utf-8 -*- """Caliopen IO interfaces definitions.""" from __future__ import absolute_import, print_function, unicode_literals import zope.interface """ marshall functions transform a core object to something expected by counterpart unMarshall func transform something coming from counterpart to a core object """ class JsonDictIO(zope.interface.Interface): """json dict is a dict ready to be serialized into a json ie : attr values are only str, int, bool or None """ def marshall_json_dict(**options): raise NotImplementedError def unmarshall_json_dict(**options): raise NotImplementedError class ProtobufIO(zope.interface.Interface): """IO between caliopen's objects and protobuf objects""" def marshall_protobuf(**options): raise NotImplementedError def unmarshall_protobuf(message, **options): raise NotImplementedError class DictIO(zope.interface.Interface): def marshall_dict(**options): raise NotImplementedError def unmarshall_dict(document, **options): raise NotImplementedError class JsonIO(zope.interface.Interface): """json is an array of bytes""" def marshall_json(**options): raise NotImplementedError def unmarshall_json(document, **options): raise NotImplementedError ================================================ FILE: src/backend/main/py.main/caliopen_main/common/interfaces/__init__.py ================================================ # -*- coding: utf-8 -*- """Caliopen main interfaces definitions.""" from __future__ import absolute_import, print_function, unicode_literals from .IO import JsonDictIO, ProtobufIO, DictIO, JsonIO from .storage import DbIO, IndexIO from .parser import IAttachmentParser, IMessageParser, IParticipantParser __all__ = ['JsonDictIO', 'ProtobufIO', 'DictIO', 'JsonIO', 'DbIO', 'IndexIO', 'IAttachmentParser', 'IMessageParser', 'IParticipantParser'] ================================================ FILE: src/backend/main/py.main/caliopen_main/common/interfaces/parser.py ================================================ # -*- coding: utf-8 -*- """Caliopen base classes for parsing logic.""" from __future__ import absolute_import, print_function, unicode_literals import zope.interface class IAttachmentParser(zope.interface.Interface): """Interface for a message attachment parsing.""" content_type = zope.interface.Attribute('Attachment MIME content type') filename = zope.interface.Attribute('Filename if any') data = zope.interface.Attribute('Attachment data') size = zope.interface.Attribute('Attachment size') charset = zope.interface.Attribute('Attachment charset') is_inline = zope.interface.Attribute('Is inline') mime_boundary = zope.interface.Attribute('MIME boundary value') class IParticipantParser(zope.interface.Interface): """Interface for a message participant parsing.""" type = zope.interface.Attribute('Participant role type') address = zope.interface.Attribute('Participant address') label = zope.interface.Attribute('Participant label') class IMessageParser(zope.interface.Interface): """Interface for all message parsers.""" message_protocol = zope.interface.Attribute('Type of message') raw = zope.interface.Attribute('Raw message') subject = zope.interface.Attribute('Message subject if any') date = zope.interface.Attribute('Message date') size = zope.interface.Attribute('Message size in bytes') body_html = zope.interface.Attribute('Message html body') body_plain = zope.interface.Attribute('Message plain txt body') participants = zope.interface.Attribute('List of participants') attachments = zope.interface.Attribute('List of attachments') external_references = zope.interface.Attribute('External references') extra_parameters = zope.interface.Attribute('Extra parameters') ================================================ FILE: src/backend/main/py.main/caliopen_main/common/interfaces/storage.py ================================================ # -*- coding: utf-8 -*- """Caliopen storage interfaces definitions.""" from __future__ import absolute_import, print_function, unicode_literals import zope.interface class DbIO(zope.interface.Interface): """Interface for objects persisted in cassandra""" def get_db(**options): """Retreive object from db and place it in a _model_class instance""" raise NotImplementedError def save_db(**options): """""" raise NotImplementedError def create_db(**options): """""" raise NotImplementedError def delete_db(**options): """""" raise NotImplementedError def update_db(**options): """Update values within _model_class from object values and save them""" raise NotImplementedError def marshall_db(**options): """Create a _model_class instance with current object attributes For now, we rely on cassandra's Encoder. We could customize this marshaller in future if needed """ raise NotImplementedError def unmarshall_db(**options): """Fill object attributes with values from _db""" raise NotImplementedError class IndexIO(zope.interface.Interface): """Interface for objects indexed in Elasticsearch""" def marshall_index(**options): """Create a _index_class instance with current object attributes""" raise NotImplementedError def unmarshall_index(**options): """Fill object's attributes with values from _index""" raise NotImplementedError ================================================ FILE: src/backend/main/py.main/caliopen_main/common/objects/__init__.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/common/objects/base.py ================================================ import zope.interface import types import uuid import datetime import pytz from six import add_metaclass from caliopen_storage.exception import NotFound from caliopen_main.common.errors import PatchConflict, PatchUnprocessable, \ PatchError from caliopen_main.common.errors import ForbiddenAction from caliopen_main.common.interfaces import (IO, storage) from elasticsearch import exceptions as ESexceptions from caliopen_storage.core.base import CoreMetaClass import logging log = logging.getLogger(__name__) class CaliopenObject(object): """ Empty class to identify Caliopen objects/types. all custom classes should inherit from that """ _attrs = {} def __init__(self, **kwargs): # TODO: type check and kwargs consistency check for k, v in kwargs.items(): if k in self._attrs: if isinstance(self._attrs[k], list): setattr(self, k, []) att_list = getattr(self, k) if isinstance(v, list): for item in v: if issubclass(self._attrs[k][0], CaliopenObject): att_list.append(self._attrs[k][0](**item)) else: att_list.append(item) else: if issubclass(self._attrs[k][0], CaliopenObject): att_list.append(self._attrs[k][0](**v)) else: att_list.append(v) elif issubclass(self._attrs[k], CaliopenObject): setattr(self, k, self._attrs[k][0](v)) else: setattr(self, k, v) for attr, attrtype in self._attrs.items(): if not hasattr(self, attr): if isinstance(attrtype, list): setattr(self, attr, []) elif isinstance(attrtype, dict): setattr(self, attr, {}) else: setattr(self, attr, None) def keys(self): """returns a list of current attributes""" return [k for k in self._attrs if hasattr(self, k)] def update_with(self, sibling): """update self attributes with those from sibling returns a list of first level attributes that have been modified """ pass class ObjectDictifiable(CaliopenObject): """Object that can marshall/unmarshall to/from python dict""" zope.interface.implements(IO.DictIO) def marshall_dict(self, **options): """output a dict representation of self 'public' attributes""" self_dict = {} for att, val in vars(self).items(): if not att.startswith("_") and val is not None: if isinstance(self._attrs[att], types.ListType): lst = [] if len(att) > 0: if issubclass(self._attrs[att][0], ObjectDictifiable): for item in val: lst.append(item.marshall_dict()) else: lst = val self_dict[att] = lst else: self_dict[att] = lst elif issubclass(self._attrs[att], ObjectDictifiable): self_dict[att] = val.marshall_dict() else: self_dict[att] = val return self_dict def unmarshall_dict(self, document, **options): """squash self.attrs with dict input document all self.attrs are reset if not in document """ for attr, attrtype in self._attrs.items(): if attr in document and document[attr] is not None: unmarshall_item(document, attr, self, attrtype, is_creation=False) else: if isinstance(attrtype, types.ListType): setattr(self, attr, []) elif issubclass(attrtype, types.DictType): setattr(self, attr, {}) elif issubclass(attrtype, types.BooleanType): setattr(self, attr, False) elif issubclass(attrtype, types.StringType): setattr(self, attr, "") elif issubclass(attrtype, types.IntType): setattr(self, attr, 0) else: setattr(self, attr, None) class ObjectJsonDictifiable(ObjectDictifiable): """Object can marshall/unmarshall to/from python json compatible dict A json compatible dict is a dict with only 4 value types : str num bool None We rely on schematics to do the job from our object's 'public' attributes """ zope.interface.implements(IO.JsonDictIO) _json_model = None def marshall_json_dict(self, **options): d = self.marshall_dict() return self._json_model(d).serialize() def unmarshall_json_dict(self, document, **options): """ TODO: handle conversion of basic json type into obj. types""" # validate document against json_model before trying to unmarshal valid_doc = self._json_model(document) try: valid_doc.validate() except Exception as exc: log.warn("document validation failed with error {}".format(exc)) raise exc self.unmarshall_dict(document, **options) @add_metaclass(CoreMetaClass) class ObjectStorable(ObjectJsonDictifiable): zope.interface.implements(storage.DbIO) _model_class = None # cql model for object _db = None # cql model instance _pkey_name = None # name of primary key in cassandra _relations = None # related tables into cassandra _lookup_class = None # _lookup_values = None # tables keys, values for lookups def get_db(self, **options): """Get a core object from database and put it in self._db attribute""" if self._pkey_name: param = { self._pkey_name: getattr(self, self._pkey_name) } else: param = {} self._db = self._model_class.get(**param) if self._db is None: raise NotFound('%s #%s not found.' % (self.__class__.__name__, getattr(self, self._pkey_name))) def save_db(self, **options): try: self._db.save() except Exception as exc: log.exception(exc) return exc return None def delete_db(self, **options): try: self._db.delete() except Exception as exc: log.exception(exc) return exc return None def update_db(self, **options): """push updated model into db""" try: self._db.update() except Exception as exc: log.exception(exc) return exc return None def marshall_db(self, **options): """squash self._db with self 'public' attributes self._db being a cqlengine Model, we can (re)set attributes one by one, cqlengine will do the job of changes logging to later make a smart update into the db """ if not isinstance(self._db, self._model_class): self._db = self._model_class() self_keys = self._attrs.keys() for att in self._db.keys(): if not att.startswith("_") and att in self_keys: # TODO : manage protected attrs # (ie attributes that user should not be able to change) if isinstance(self._attrs[att], list): # TODO : manage change within list to only elem changed # (use builtin set() collection ?) if issubclass(self._attrs[att][0], CaliopenObject): setattr(self._db, att, [self._attrs[att][0]._model_class( **x.marshall_dict()) for x in getattr(self, att)]) else: setattr(self._db, att, getattr(self, att)) else: if issubclass(self._attrs[att], datetime.datetime) and \ getattr(self, att) is not None: # datetime in cqlengine are 'naive', ours are 'aware' setattr(self._db, att, getattr(self, att).replace(tzinfo=None)) else: self_att = self._attrs[att] get_att = getattr(self, att) if issubclass(self_att, CaliopenObject): if get_att is not None: setattr(self._db, att, self_att._model_class( **get_att.marshall_dict())) else: setattr(self._db, att, get_att) def unmarshall_db(self, **options): """squash self.attrs with db representation""" if isinstance(self._db, self._model_class): self.unmarshall_dict(dict(self._db)) else: log.warn('Invalid model class, expect {}, have {}'. format(self._db.__class__, self._model_class.__class__)) def set_uuid(self): setattr(self, self._model_class._pkey, uuid.uuid4()) class ObjectUser(ObjectStorable): """Objects that MUST belong to a user to survive in Caliopen's world...""" def __init__(self, user=None, **params): self.user = user if user: self.user_id = user.user_id else: self.user_id = None super(ObjectUser, self).__init__(**params) def marshall_dict(self, **options): """output a dict representation of self 'public' attributes""" self_dict = {} for att, val in vars(self).items(): if not att.startswith("_") and val is not None and att != 'user': if isinstance(self._attrs[att], types.ListType): lst = [] if len(att) > 0: if issubclass(self._attrs[att][0], ObjectDictifiable): for item in val: lst.append(item.marshall_dict()) else: lst = val self_dict[att] = lst else: self_dict[att] = lst elif issubclass(self._attrs[att], ObjectDictifiable): self_dict[att] = val.marshall_dict() else: self_dict[att] = val return self_dict @classmethod def list_db(cls, user): """List all objects that belong to an user.""" models = cls._model_class.filter(user_id=user.user_id) objects = [] for model in models: obj = cls(user) obj._db = model obj.unmarshall_db() objects.append(obj) return objects def get_db(self, **options): """Get an object belonging to an user and put it in self._db attrs""" if self._pkey_name: param = { self._pkey_name: getattr(self, self._pkey_name) } else: param = {} try: self._db = self._model_class.get(user_id=self.user_id, **param) except NotFound: raise NotFound('%s %s not found for user %s' % (self.__class__.__name__, param[self._pkey_name], self.user_id)) def apply_patch(self, patch, **options): """ Update self attributes with patch rfc7396 and Caliopen's specifications if, and only if, patch is consistent with current obj db instance :param patch: json-dict object describing the patch to apply with a "current_state" key. see caliopen rfc for explanation :param options: whether patch should be propagated to db and/or index :return: Exception or None """ if patch is None or "current_state" not in patch: raise PatchUnprocessable(message='Invalid patch') patch_current = patch.pop("current_state") # build 3 siblings : 2 from patch and last one from db obj_patch_new = self.__class__(user_id=self.user_id) obj_patch_old = self.__class__(user_id=self.user_id) try: obj_patch_new.unmarshall_json_dict(patch) except Exception as exc: log.exception(exc) raise PatchUnprocessable(message="unmarshall patch " "error: %r" % exc) try: obj_patch_old.unmarshall_json_dict(patch_current) except Exception as exc: log.exception(exc) raise PatchUnprocessable(message="unmarshall current " "patch error: %r" % exc) self.get_db() # TODO : manage protected attributes, to prevent patch on them if "tags" in patch.keys(): raise ForbiddenAction( message="patching tags through parent object is forbidden") # check if patch is consistent with db current state # if it is, squash self attributes self.unmarshall_db() for key in patch.keys(): current_attr = self._attrs[key] try: self._check_key_consistency(current_attr, key, obj_patch_old, obj_patch_new) except Exception as exc: log.exception("key consistency checking failed: {}". format(exc)) raise exc # all controls passed, we can actually set the new attribute create_sub_object = False if key not in patch_current.keys(): create_sub_object = True else: if patch_current[key] in (None, [], {}): create_sub_object = True if isinstance(patch_current[key], list) and len( patch[key]) > len(patch_current[key]): create_sub_object = True if patch[key] is not None: unmarshall_item(patch, key, self, self._attrs[key], create_sub_object) if "db" in options and options["db"] is True: # apply changes to db model and update db if "with_validation" in options and options[ "with_validation"] is True: d = self.marshall_dict() try: self._json_model(d).validate() except Exception as exc: log.exception("document is not valid: {}".format(exc)) raise PatchUnprocessable( message="document is not valid," " can't insert it into db: <{}>".format(exc)) self.marshall_db() try: self.update_db() except Exception as exc: log.exception(exc) raise PatchError(message="Error when updating db") def _check_key_consistency(self, current_attr, key, obj_patch_old, patch_current): """ check if a key provided in patch is consistent with current state """ if key not in self._attrs.keys(): raise PatchUnprocessable( message="unknown key in patch") old_val = getattr(obj_patch_old, key) cur_val = getattr(self, key) msg = "Patch current_state not consistent with db, step {} key {}" if isinstance(current_attr, types.ListType): if not isinstance(cur_val, types.ListType): raise PatchConflict( messag=msg.format(0, key)) if key not in patch_current.keys(): # means patch wants to add the key. # Value in db should be null or empty if cur_val not in (None, [], {}): raise PatchConflict( message=msg.format(0.5, key)) else: if isinstance(current_attr, types.ListType): if old_val == [] and cur_val != []: raise PatchConflict( message=msg.format(1, key)) if cur_val == [] and old_val != []: raise PatchConflict( message=msg.format(2, key)) for old in old_val: for elem in cur_val: if issubclass(current_attr[0], CaliopenObject): if elem.__dict__ == old.__dict__: break else: if elem == old: break else: raise PatchConflict( message=msg.format(3, key)) elif issubclass(self._attrs[key], types.DictType): if cmp(old_val, cur_val) != 0: raise PatchConflict( message=msg.format(4, key)) else: # XXX ugly patch but else compare 2 distinct objects # and not their representation if hasattr(old_val, 'marshall_dict') and \ hasattr(cur_val, 'marshall_dict'): old_val = old_val.marshall_dict() cur_val = cur_val.marshall_dict() if old_val != cur_val: raise PatchConflict( message=msg.format(5, key)) class ObjectIndexable(ObjectUser): zope.interface.implements(storage.IndexIO) _index_class = None # dsl model for object _index = None # dsl model instance def get_index(self, **options): """Get a doc from ES within user's index and put it at self._index""" obj_id = getattr(self, self._pkey_name) try: self._index = self._index_class.get(index=self.user.shard_id, id=obj_id, using=self._index_class.client()) except Exception as exc: if isinstance(exc, ESexceptions.NotFoundError): log.exception("indexed doc not found") self._index = None raise NotFound('%s #%s not found for user %s' % (self.__class__.__name__, obj_id, self.user_id)) else: raise exc def save_index(self, wait_for=False, **options): if wait_for: self._index.save(using=self._index_class.client(), refresh="wait_for") else: self._index.save(using=self._index_class.client()) def create_index(self, **options): """Create indexed document from current self._index state""" self.marshall_index() self.save_index() def delete_index(self, **options): try: self._index.delete(using=self._index_class.client(), refresh="wait_for") except Exception as exc: log.exception(exc) return exc return None def update_index(self, wait_for=False, **options): """get indexed doc from elastic and update it with self attrs if indexed doc doesn't exist, create it else update changed fields only """ self.get_index() if self._index is not None: try: update_dict = self.marshall_index(update=True) if wait_for: self._index.update(using=self._index_class.client(), refresh="wait_for", **update_dict) else: self._index.update(using=self._index_class.client(), **update_dict) except Exception as exc: log.exception("update index failed: {}".format(exc)) else: # for some reasons, index doc not found... create one from scratch self.create_index() def marshall_index(self, **options): """squash self._index with self 'public' attributes options: update=True : only changed values will be replace in self._index and a dict with changed applied will be returned """ # TODO : manage protected attrs (ie attributes that user should not be able to change directly) update = False if "update" in options and options["update"] is True: update = True # index_sibling is instanciated with self._index values to perform # object comparaison index_sibling = self.__class__(user=self.user) index_sibling._index = self._index index_sibling.unmarshall_index() if not isinstance(self._index, self._index_class): self._index = self._index_class() self._index.meta.index = self.user.shard_id self._index.meta.using = self._index.client() self._index.meta.id = getattr(self, self._pkey_name) # update_sibling is an empty sibling that will be filled # with attributes from self update_sibling = self.__class__(user=self.user) m = self._index._doc_type.mapping.to_dict() for att in m[self._index._doc_type.name]["properties"]: if not att.startswith("_") and att in index_sibling.keys(): if update: if getattr(self, att) != getattr(index_sibling, att): setattr(update_sibling, att, getattr(self, att)) else: delattr(update_sibling, att) else: setattr(update_sibling, att, getattr(self, att)) update_dict = update_sibling.marshall_dict() for k, v in update_dict.iteritems(): if k in self._index_class.__dict__: # do not try to set a property directly if not isinstance(getattr(self._index_class, k), property): setattr(self._index, k, v) else: setattr(self._index, k, v) if update: return update_sibling.marshall_dict() def unmarshall_index(self, **options): """squash self.attrs with index representation""" if isinstance(self._index, self._index_class): self.unmarshall_dict(self._index.to_dict()) def apply_patch(self, patch, **options): try: super(ObjectIndexable, self).apply_patch(patch, **options) except Exception as exc: log.exception("ObjectIndexable apply_patch() returned error: {}". format(exc)) raise exc if "index" in options and options["index"] is True: # silently update index. Should we raise an error if it fails ? try: self.update_index(wait_for=True) except Exception as exc: log.exception("apply_patch update_index() exception: {}". format(exc)) raise exc def unmarshall_item(document, key, target_object, target_attr_type, is_creation): """ general function to cast a dict item (ie: document[key]) into the corresponding target_object's attr (ie: target_object.key) :param document: source dict :param key: source dict key to unmarshall :param target_object: object to unmarshall document[key] into :param target_attr_type: the types.type of corresponding attr in target obj. :param is_creation: if true, we are in the context of the creation of an obj :return: nothing, target object is modified in-place """ if isinstance(target_attr_type, list): lst = [] if issubclass(target_attr_type[0], ObjectDictifiable): for item in document[key]: sub_obj = target_attr_type[0]() sub_obj.unmarshall_dict(item) if is_creation and isinstance(sub_obj, ObjectStorable): sub_obj.set_uuid() lst.append(sub_obj) elif issubclass(target_attr_type[0], uuid.UUID): for item in document[key]: sub_obj = uuid.UUID(str(item)) lst.append(sub_obj) else: lst = document[key] setattr(target_object, key, lst) elif issubclass(target_attr_type, ObjectDictifiable): if hasattr(target_object, 'user'): opts = {'user': target_object.user} else: opts = {} sub_obj = target_attr_type(**opts) sub_obj.unmarshall_dict(document[key]) setattr(target_object, key, sub_obj) elif issubclass(target_attr_type, uuid.UUID): setattr(target_object, key, uuid.UUID(str(document[key]))) elif issubclass(target_attr_type, datetime.datetime): if document[key] is not None \ and document[key].tzinfo is None: setattr(target_object, key, document[key].replace(tzinfo= pytz.utc)) else: setattr(target_object, key, document[key]) else: new_attr = document[key] if hasattr(target_attr_type, "validate"): new_attr = target_attr_type().validate(document[key]) setattr(target_object, key, new_attr) ================================================ FILE: src/backend/main/py.main/caliopen_main/common/objects/tag.py ================================================ # -*- coding: utf-8 -*- """Caliopen User tag parameters classes.""" from __future__ import absolute_import, print_function, unicode_literals import types import datetime from .base import ObjectJsonDictifiable from ..store.tag import ResourceTag as ModelResourceTag from ..store.tag import IndexedResourceTag import logging log = logging.getLogger(__name__) ### legacy code. # Tags are not anymore nested into other objects as objects, but as []string. class ResourceTag(ObjectJsonDictifiable): """Tag nested in resources.""" _attrs = { 'date_insert': datetime.datetime, 'importance_level': types.IntType, 'name': types.StringType, 'label': types.StringType, 'type': types.StringType, } _model_class = ModelResourceTag _pkey_name = 'tag_id' _index_class = IndexedResourceTag ================================================ FILE: src/backend/main/py.main/caliopen_main/common/parameters/__init__.py ================================================ # -*- coding: utf-8 -*- """Caliopen common parameters classes.""" from __future__ import absolute_import, print_function, unicode_literals from .pubkey import NewPublicKey, PublicKey __all__ = ['NewPublicKey', 'PublicKey'] ================================================ FILE: src/backend/main/py.main/caliopen_main/common/parameters/pubkey.py ================================================ # -*- coding: utf-8 -*- """Caliopen contact parameters classes.""" from __future__ import absolute_import, print_function, unicode_literals from schematics.models import Model from schematics.types import StringType, UUIDType, DateTimeType, LongType from schematics.transforms import blacklist import caliopen_storage.helpers.json as helpers KEY_CHOICES = ['rsa', 'gpg', 'ssh'] class NewPublicKey(Model): """Input structure for a new public key.""" resource_id = UUIDType(required=True) resource_type = StringType(required=True) key_id = UUIDType() expire_date = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') label = StringType(required=True) fingerprint = StringType() key = StringType() type = StringType() # JWT parameters kty = StringType() # rsa / ec use = StringType() # sig / enc alg = StringType() # algorithm # Elliptic curve public key parameters (rfc7518 6.2.1) crv = StringType() x = LongType() y = LongType() class Options: serialize_when_none = False class PublicKey(NewPublicKey): """Existing public key.""" key_id = UUIDType(required=True) date_insert = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') date_update = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') user_id = UUIDType() class Options: roles = {'default': blacklist('user_id', 'device_id')} serialize_when_none = False ================================================ FILE: src/backend/main/py.main/caliopen_main/common/parameters/tag.py ================================================ # -*- coding: utf-8 -*- """Caliopen tags parameters.""" from schematics.models import Model from schematics.types import StringType, UUIDType, DateTimeType, IntType class ResourceTag(Model): """Tag related to a resource.""" date_insert = DateTimeType() importance_level = IntType() name = StringType() tag_id = UUIDType() type = StringType() ================================================ FILE: src/backend/main/py.main/caliopen_main/common/parameters/types.py ================================================ # -*- coding: utf-8 -*- """Caliopen contact parameter validators.""" from __future__ import absolute_import, print_function, unicode_literals from schematics.types import StringType from schematics.exceptions import ValidationError import phonenumbers from ..helpers.normalize import clean_email_address class InternetAddressType(StringType): """Validate an email or instant messaging address, return normalized.""" def validate_email(self, value, context=None): try: clean, email = clean_email_address(value) except Exception as exc: raise ValidationError(exc.message) return clean class PhoneNumberType(StringType): """Validate a phone number and normalize in international format.""" def validate_phone(self, value, context=None): try: number = phonenumbers.parse(value, None) phone_format = phonenumbers.PhoneNumberFormat.INTERNATIONAL return phonenumbers.format_number(number, phone_format) except Exception as exc: raise ValidationError(exc.message) ================================================ FILE: src/backend/main/py.main/caliopen_main/common/store/__init__.py ================================================ # -*- coding: utf-8 -*- """Caliopen common store classes classes.""" from __future__ import absolute_import, print_function, unicode_literals from .pubkey import PublicKey __all__ = ['PublicKey'] ================================================ FILE: src/backend/main/py.main/caliopen_main/common/store/pubkey.py ================================================ # -*- coding: utf-8 -*- """Caliopen objects related to contact definition.""" from __future__ import absolute_import, print_function, unicode_literals from datetime import datetime from cassandra.cqlengine import columns from caliopen_storage.store import BaseModel class PublicKey(BaseModel): """Public cryptographic keys model.""" user_id = columns.UUID(primary_key=True) resource_id = columns.UUID(primary_key=True) # clustering key key_id = columns.UUID(primary_key=True) # clustering key resource_type = columns.Text() label = columns.Text() date_insert = columns.DateTime(default=datetime.utcnow) date_update = columns.DateTime() expire_date = columns.DateTime() emails = columns.List(columns.Text()) key = columns.Text() fingerprint = columns.Text() size = columns.VarInt() # JWT parameters kty = columns.Text() # rsa / ec use = columns.Text() # sig / enc alg = columns.Text() # algorithm # Elliptic curve public key parameters (rfc7518 6.2.1) crv = columns.Text() x = columns.VarInt() y = columns.VarInt() ================================================ FILE: src/backend/main/py.main/caliopen_main/common/store/tag.py ================================================ # -*- coding: utf-8 -*- """Caliopen tag objects.""" from __future__ import absolute_import, print_function, unicode_literals from cassandra.cqlengine import columns from elasticsearch_dsl import InnerObjectWrapper, Date, Integer from elasticsearch_dsl import Boolean, Keyword from caliopen_storage.store import BaseUserType class ResourceTag(BaseUserType): """Tag nested in resource model.""" _pkey = 'tag_id' date_insert = columns.DateTime() importance_level = columns.Integer() name = columns.Text() tag_id = columns.UUID() type = columns.Text() class IndexedResourceTag(InnerObjectWrapper): """Nested tag into indexed resource model.""" date_insert = Date() importance_level = Integer() name = Keyword() tag_id = Keyword() type = Boolean() ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/__init__.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/core.py ================================================ # -*- coding: utf-8 -*- """Caliopen contact core classes.""" from __future__ import absolute_import, print_function, unicode_literals import logging import uuid import datetime import pytz import phonenumbers from .store import (Contact as ModelContact, ContactLookup as ModelContactLookup, Organization, Email, IM, PostalAddress, Phone, SocialIdentity) from .store.contact_index import IndexedContact from caliopen_storage.core import BaseCore from caliopen_storage.exception import NotFound from caliopen_storage.core.mixin import MixinCoreRelation, MixinCoreNested from caliopen_main.pi.objects import PIModel from caliopen_main.common.core import BaseUserCore from caliopen_main.participant.objects import Participant log = logging.getLogger(__name__) class ContactLookup(BaseUserCore): """Contact lookup core class.""" _model_class = ModelContactLookup _pkey_name = 'value' class BaseContactSubCore(BaseCore): """ Base core object for contact related objects """ @classmethod def create(cls, user, contact, **kwargs): obj = cls._model_class.create(user_id=user.user_id, contact_id=contact.contact_id, **kwargs) return cls(obj) @classmethod def get(cls, user, contact, value): kwargs = {cls._pkey_name: value} try: obj = cls._model_class.get(user_id=user.user_id, contact_id=contact.contact_id, **kwargs) return cls(obj) except Exception: return None @classmethod def find(cls, user, contact, filters=None): q = cls._model_class.filter(user_id=user.user_id). \ filter(contact_id=contact.contact_id) if not filters: objs = q else: objs = q.filter(**filters) return {'total': len(objs), 'data': [cls(x) for x in objs]} def to_dict(self): return {col: getattr(self, col) for col in self._model_class._columns.keys()} class Contact(BaseUserCore, MixinCoreRelation, MixinCoreNested): _model_class = ModelContact _pkey_name = 'contact_id' _index_class = IndexedContact _nested = { 'emails': Email, 'phones': Phone, 'ims': IM, 'social_identities': SocialIdentity, 'addresses': PostalAddress, 'organizations': Organization, } # Any of these nested objects,can be a lookup value _lookup_class = ContactLookup _lookup_values = { 'emails': {'value': 'address', 'type': 'email'}, 'ims': {'value': 'address', 'type': 'email'}, 'phones': {'value': 'number', 'type': 'phone'}, 'social_identities': {'value': 'name', 'type': 'social'}, } @classmethod def _compute_title(cls, contact): elmts = [] elmts.append(contact.name_prefix) if contact.name_prefix else None elmts.append(contact.name_suffix) if contact.name_suffix else None elmts.append(contact.given_name) if contact.given_name else None elmts.append(contact.additional_name) if \ contact.additional_name else None elmts.append(contact.family_name) if contact.family_name else None # XXX may be empty, got info from related infos return " ".join(elmts) def _create_lookup(self, type, value): """Create one contact lookup.""" log.debug('Will create lookup for type {} and value {}'. format(type, value)) lookup = ContactLookup.create(self.user, value=value, type=type, contact_id=self.contact_id) participant = Participant(address=value, protocol=type) return lookup def _create_lookups(self): """Create lookups for a contact using its nested attributes.""" for attr_name, obj in self._lookup_values.items(): nested = getattr(self, attr_name) if nested: for attr in nested: lookup_value = attr[obj['value']] if lookup_value: self._create_lookup(obj['type'], lookup_value) @classmethod def normalize_phones(cls, phones): for phone in phones: try: normalized = phonenumbers.parse(phone.number, None) phone_format = phonenumbers.PhoneNumberFormat.INTERNATIONAL new = phonenumbers.format_number(normalized, phone_format) phone.normalized_number = new except: pass @classmethod def create(cls, user, contact, **related): # XXX do sanity check about only one primary for related objects # XXX check no extra arguments in related than relations contact.validate() for k, v in related.iteritems(): if k in cls._relations: [x.validate() for x in v] else: raise Exception('Invalid argument to contact.create : %s' % k) contact_id = uuid.uuid4() if not contact.title: title = cls._compute_title(contact) else: title = contact.title if not contact.given_name and not contact.family_name: # XXX more complex logic and not arbitrary order and character if ',' in contact.title: gn, fn = contact.title.split(',', 2) contact.given_name = gn.rstrip().lstrip() contact.family_name = fn.rstrip().lstrip() # XXX PI compute pi = PIModel() pi.technic = 0 pi.comportment = 0 pi.context = 0 pi.version = 0 phones = cls.create_nested(contact.phones, Phone) # Normalize phones if possible cls.normalize_phones(phones) attrs = {'contact_id': contact_id, 'info': contact.infos, 'groups': contact.groups, 'date_insert': datetime.datetime.now(tz=pytz.utc), 'given_name': contact.given_name, 'additional_name': contact.additional_name, 'family_name': contact.family_name, 'prefix_name': contact.name_prefix, 'suffix_name': contact.name_suffix, 'title': title, 'emails': cls.create_nested(contact.emails, Email), 'ims': cls.create_nested(contact.ims, IM), 'phones': phones, 'addresses': cls.create_nested(contact.addresses, PostalAddress), 'social_identities': cls.create_nested(contact.identities, SocialIdentity), 'organizations': cls.create_nested(contact.organizations, Organization), 'tags': contact.tags, 'pi': pi} core = super(Contact, cls).create(user, **attrs) log.debug('Created contact %s' % core.contact_id) core._create_lookups() # Create relations related_cores = {} for k, v in related.iteritems(): if k in cls._relations: for obj in v: log.debug('Processing object %r' % obj) # XXX check only one is_primary per relation using it new_core = cls._relations[k].create(user, core, **obj) related_cores.setdefault(k, []).append(new_core.to_dict()) log.debug('Created related core %r' % new_core) return core @classmethod def lookup(cls, user, value): try: lookup = ContactLookup._model_class.get(user_id=user.user_id, value=value) except NotFound: return None if lookup and lookup.contact_id: # as of 2019, april it is forbidden in Caliopen # to add an external address to more than one contact # as a mater of fact, lookup should always return one contact only try: return cls.get(user, lookup.contact_id) except NotFound: log.warn('Inconsistent contact lookup with non existing ' ' contact %r' % lookup.contact_id) return None # XXX something else to do ? return None def delete(self): if self.user.contact_id == self.contact_id: raise Exception("Can't delete contact related to user") return super(Contact, self).delete() @property def public_keys(self): """Return detailed public keys.""" return self._expand_relation('public_keys') # MixinCoreRelation methods def add_organization(self, organization): return self._add_nested('organizations', organization) def delete_organization(self, organization_id): return self._delete_nested('organizations', organization_id) def add_address(self, address): return self._add_nested('addresses', address) def delete_address(self, address_id): return self._delete_nested('addresses', address_id) def add_email(self, email): return self._add_nested('emails', email) def delete_email(self, email_addr): return self._delete_nested('emails', email_addr) def add_im(self, im): return self._add_nested('ims', im) def delete_im(self, im_addr): return self._delete_nested('ims', im_addr) def add_phone(self, phone): return self._add_nested('phones', phone) def delete_phone(self, phone_num): return self._delete_nested('phones', phone_num) def add_social_identity(self, identity): return self._add_nested('social_identities', identity) def delete_social_identity(self, identity_name): return self._delete_nested('social_identities', identity_name) def add_public_key(self, key): # XXX Compute fingerprint and check key validity return self._add_relation('public_keys', key) def delete_public_key(self, key_id): return self._delete_relation('public_keys', key_id) ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/objects/__init__.py ================================================ # -*- coding: utf-8 -*- """Caliopen contact classes.""" from __future__ import absolute_import, print_function, unicode_literals from .contact import Contact __all__ = ['Contact'] ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/objects/contact.py ================================================ # -*- coding: utf-8 -*- """Caliopen contact parameters classes.""" from __future__ import absolute_import, print_function, unicode_literals import types from uuid import UUID import datetime from caliopen_main.common.objects.base import ObjectStorable, ObjectIndexable from ..store.contact import (Contact as ModelContact, ContactLookup as ModelContactLookup) from ..store.contact_index import IndexedContact from ..parameters import Contact as ParamContact from .email import Email from .identity import SocialIdentity from .im import IM from .organization import Organization from .phone import Phone from .postal_address import PostalAddress from caliopen_main.pi.objects import PIObject from caliopen_storage.exception import NotFound from caliopen_main.common.errors import ForbiddenAction import logging log = logging.getLogger(__name__) class ContactLookup(ObjectStorable): """Contact lookup core class.""" def __init__(self): self._model_class = ModelContactLookup self._pkey_name = 'value' class Contact(ObjectIndexable): # TODO : manage attrs that should not be modifiable directly by users _attrs = { 'additional_name': types.StringType, 'addresses': [PostalAddress], 'avatar': types.StringType, 'contact_id': UUID, 'date_insert': datetime.datetime, 'date_update': datetime.datetime, 'deleted': datetime.datetime, 'emails': [Email], 'family_name': types.StringType, 'given_name': types.StringType, 'groups': [types.StringType], 'identities': [SocialIdentity], 'ims': [IM], 'infos': types.DictType, 'name_prefix': types.StringType, 'name_suffix': types.StringType, 'organizations': [Organization], 'phones': [Phone], 'pi': PIObject, 'privacy_features': types.DictType, 'tags': [types.StringType], 'title': types.StringType, 'user_id': UUID } _json_model = ParamContact # operations related to cassandra _model_class = ModelContact _db = None # model instance with datas from db _pkey_name = "contact_id" _lookup_class = ContactLookup _lookup_values = { 'emails': {'value': 'address', 'type': 'email'}, 'ims': {'value': 'address', 'type': 'email'}, 'phones': {'value': 'number', 'type': 'phone'}, 'social_identities': {'value': 'name', 'type': 'social'}, } # operations related to elasticsearch _index_class = IndexedContact _index = None def delete(self): # XXX prevent circular dependency import from caliopen_main.user.core.user import User user = User.get(self.user_id) if user.contact_id == self.contact_id: raise ForbiddenAction("Can't delete contact related to user") try: self.get_db() self.get_index() except Exception as exc: raise NotFound try: self.delete_db() self.delete_index() except Exception as exc: raise exc @classmethod def _compute_title(cls, contact): elmts = [] elmts.append(contact.name_prefix) if contact.name_prefix else None elmts.append(contact.name_suffix) if contact.name_suffix else None elmts.append(contact.given_name) if contact.given_name else None elmts.append(contact.additional_name) if \ contact.additional_name else None elmts.append(contact.family_name) if contact.family_name else None return " ".join(elmts) if len(elmts) > 0 else " (N/A) " ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/objects/email.py ================================================ # -*- coding: utf-8 -*- """Caliopen contact parameters classes.""" from __future__ import absolute_import, print_function, unicode_literals import types from caliopen_main.common.objects.base import ObjectStorable from uuid import UUID from caliopen_main.common.parameters.types import InternetAddressType from ..parameters import Email as EmailParam from ..store.contact import Email as ModelEmail class Email(ObjectStorable): _attrs = { "address": InternetAddressType, "email_id": UUID, "is_primary": types.BooleanType, "label": types.StringType, "type": types.StringType, } _json_model = EmailParam _model_class = ModelEmail ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/objects/identity.py ================================================ # -*- coding: utf-8 -*- """Caliopen message object classes.""" from __future__ import absolute_import, print_function, unicode_literals import types from uuid import UUID from caliopen_main.common.objects.base import ObjectIndexable, \ ObjectJsonDictifiable from caliopen_main.pi.objects import PIObject from ..store.contact import SocialIdentity as ModelSocialIdentity from ..parameters import SocialIdentity as SocialIdentityParam from ..store.contact_index import IndexedSocialIdentity class SocialIdentity(ObjectIndexable): """Social identity related to a contact.""" _attrs = { "contact_id": UUID, "social_id": UUID, "infos": types.DictType, "name": types.StringType, "type": types.StringType, "user_id": UUID } _json_model = SocialIdentityParam _model_class = ModelSocialIdentity _index_class = IndexedSocialIdentity class ContactIdentity(ObjectJsonDictifiable): """ Mean of communication for a contact, with on-demand calculated PI. [for ex., a list of ContactIdentity is built for REST API …/contact/{contact_id}/identities] """ _attrs = { "identifier": types.StringType, "label": types.StringType, "privacy_index": PIObject, "protocol": types.StringType, } ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/objects/im.py ================================================ # -*- coding: utf-8 -*- """Caliopen contact parameters classes.""" from __future__ import absolute_import, print_function, unicode_literals import types from caliopen_main.common.objects.base import ObjectIndexable from uuid import UUID from caliopen_main.common.parameters.types import InternetAddressType from ..store.contact import IM as ModelIM from ..returns import IMParam from ..store.contact_index import IndexedInternetAddress class IM(ObjectIndexable): _attrs = { "address": InternetAddressType, "is_primary": types.BooleanType, "label": types.StringType, "protocol": types.StringType, "type": types.StringType, "contact_id": UUID, "im_id": UUID, "user_id": UUID } _model_class = ModelIM _json_model = IMParam _index_class = IndexedInternetAddress ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/objects/organization.py ================================================ # -*- coding: utf-8 -*- """Caliopen contact parameters classes.""" from __future__ import absolute_import, print_function, unicode_literals import types from caliopen_main.common.objects.base import ObjectIndexable from uuid import UUID from ..store.contact import Organization as ModelOrganization from ..store.contact_index import IndexedOrganization from ..returns import OrganizationParam class Organization(ObjectIndexable): _attrs = { "department": types.StringType, "is_primary": types.BooleanType, "job_description": types.StringType, "label": types.StringType, "name": types.StringType, "title": types.StringType, "type": types.StringType, "contact_id": UUID, "deleted": types.BooleanType, "organization_id": UUID, "user_id": UUID } _model_class = ModelOrganization _json_model = OrganizationParam _index_class = IndexedOrganization ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/objects/phone.py ================================================ # -*- coding: utf-8 -*- """Caliopen contact parameters classes.""" from __future__ import absolute_import, print_function, unicode_literals import logging import types from uuid import UUID import phonenumbers from caliopen_main.common.objects.base import ObjectIndexable from caliopen_main.common.parameters.types import PhoneNumberType from ..store.contact import Phone as ModelPhone from ..store.contact_index import IndexedPhone from ..returns import PhoneParam log = logging.getLogger(__name__) class Phone(ObjectIndexable): _attrs = { "contact_id": UUID, "is_primary": types.BooleanType, "number": types.StringType, "normalized_number": types.StringType, "phone_id": UUID, "type": types.StringType, "uri": types.StringType, "user_id": UUID } _model_class = ModelPhone _json_model = PhoneParam _index_class = IndexedPhone def normalize_number(self, number): try: normalized = phonenumbers.parse(number, None) phone_format = phonenumbers.PhoneNumberFormat.INTERNATIONAL return phonenumbers.format_number(normalized, phone_format) except Exception as exc: log.warn('Unable to normalize phone number {0} : {1}'. format(number, exc)) def unmarshall_dict(self, document, **options): """try to extract a normalize phone number from document""" super(Phone, self).unmarshall_dict(document, **options) normalized = self.normalize_number(self.number) if normalized: setattr(self, 'normalized_number', normalized) ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/objects/postal_address.py ================================================ # -*- coding: utf-8 -*- """Caliopen contact parameters classes.""" from __future__ import absolute_import, print_function, unicode_literals import types from caliopen_main.common.objects.base import ObjectIndexable from uuid import UUID from ..store.contact import PostalAddress as ModelPostalAddress from ..returns import PostalAddressParam from ..store.contact_index import IndexedPostalAddress class PostalAddress(ObjectIndexable): _attrs = { "address_id": UUID, "city": types.StringType, "contact_id": UUID, "country": types.StringType, "is_primary": types.BooleanType, "label": types.StringType, "postal_code": types.StringType, "region": types.StringType, "street": types.StringType, "type": types.StringType, "user_id": UUID } _model_class = ModelPostalAddress _json_model = PostalAddressParam _index_class = IndexedPostalAddress ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/parameters.py ================================================ # -*- coding: utf-8 -*- """Caliopen contact parameters classes.""" from __future__ import absolute_import, print_function, unicode_literals from schematics.models import Model from schematics.types import StringType, UUIDType, DateTimeType, BooleanType from schematics.types.compound import ListType, ModelType, DictType from schematics.transforms import blacklist from caliopen_main.common.parameters.types import InternetAddressType from caliopen_main.pi.parameters import PIParameter import caliopen_storage.helpers.json as helpers ORG_TYPES = ['work', 'home', ''] ADDRESS_TYPES = ['work', 'home', 'other', ''] EMAIL_TYPES = ['work', 'home', 'other', ''] IM_TYPES = ['work', 'home', 'other', 'netmeeting', ''] PHONE_TYPES = ['assistant', 'callback', 'car', 'company_main', 'fax', 'home', 'home_fax', 'isdn', 'main', 'mobile', 'other', 'other_fax', 'pager', 'radio', 'telex', 'tty_tdd', 'work', 'work_fax', 'work_mobile', 'work_pager'] # XXX : use configuration instead ? SOCIAL_TYPES = ['facebook', 'twitter', 'google', 'github', 'bitbucket', 'linkedin', 'ello', 'instagram', 'tumblr', 'skype', 'mastodon'] RECIPIENT_TYPES = ['to', 'from', 'cc', 'bcc'] class Recipient(Model): """Store a contact reference and one of it's address used in a message.""" address = StringType(required=True) contact_id = UUIDType() type = StringType(required=True, choices=RECIPIENT_TYPES) class Options: serialize_when_none = False class NewOrganization(Model): """Input structure for a new organization.""" department = StringType() is_primary = BooleanType(default=False) job_description = StringType() label = StringType() name = StringType(required=True) title = StringType() # XXX Add enumerated list type = StringType() class Options: serialize_when_none = False class Organization(NewOrganization): """Existing organization.""" contact_id = UUIDType() deleted = BooleanType(default=False) organization_id = UUIDType() user_id = UUIDType() class Options: roles = {'default': blacklist('user_id', 'contact_id')} serialize_when_none = False class NewPostalAddress(Model): """Input structure for a new postal address.""" address_id = StringType() city = StringType() country = StringType() is_primary = BooleanType(default=False) label = StringType() postal_code = StringType() region = StringType() street = StringType() type = StringType(choices=ADDRESS_TYPES) class Options: serialize_when_none = False class PostalAddress(NewPostalAddress): """Existing postal address.""" address_id = UUIDType() contact_id = UUIDType() user_id = UUIDType() class Options: roles = {'default': blacklist('user_id', 'contact_id')} serialize_when_none = False class NewEmail(Model): """Input structure for a new email.""" address = InternetAddressType(required=True) is_primary = BooleanType(default=False) label = StringType() type = StringType(choices=EMAIL_TYPES, default='other') class Options: serialize_when_none = False class Email(NewEmail): """Existing email.""" email_id = UUIDType() class Options: roles = {'default': blacklist('user_id', 'contact_id')} serialize_when_none = False class NewIM(Model): """Input structure for a new IM.""" address = StringType(required=True) is_primary = BooleanType(default=False) label = StringType() protocol = StringType() type = StringType(choices=IM_TYPES, default='other') class Options: serialize_when_none = False class IM(NewIM): """Existing IM.""" contact_id = UUIDType() im_id = UUIDType() user_id = UUIDType() class Options: roles = {'default': blacklist('user_id', 'contact_id')} serialize_when_none = False class NewPhone(Model): """Input structure for a new phone.""" is_primary = BooleanType(default=False) number = StringType(required=True) normalized_number = StringType() type = StringType(choices=PHONE_TYPES, default='other') uri = StringType() class Options: serialize_when_none = False class Phone(NewPhone): """Existing phone.""" contact_id = UUIDType() phone_id = UUIDType() user_id = UUIDType() class Options: roles = {'default': blacklist('user_id', 'contact_id')} serialize_when_none = False class NewSocialIdentity(Model): """Input structure for a new social identity.""" infos = DictType(StringType, default=lambda: {}) name = StringType(required=True) type = StringType(choices=SOCIAL_TYPES, required=True) class Options: serialize_when_none = False class SocialIdentity(NewSocialIdentity): """Existing social identity.""" contact_id = UUIDType() social_id = UUIDType() user_id = UUIDType() class Options: roles = {'default': blacklist('user_id', 'contact_id')} serialize_when_none = False class NewContact(Model): """Input structure for a new contact.""" additional_name = StringType() addresses = ListType(ModelType(NewPostalAddress), default=lambda: []) emails = ListType(ModelType(NewEmail), default=lambda: []) family_name = StringType() given_name = StringType() title = StringType() groups = ListType(StringType()) identities = ListType(ModelType(NewSocialIdentity), default=lambda: []) ims = ListType(ModelType(NewIM), default=lambda: [], ) infos = DictType(StringType()) name_prefix = StringType() name_suffix = StringType() organizations = ListType(ModelType(NewOrganization), default=lambda: []) phones = ListType(ModelType(NewPhone), default=lambda: []) privacy_features = DictType(StringType(), default=lambda: {}) tags = ListType(StringType(), default=lambda: []) class Options: serialize_when_none = False class Contact(NewContact): """Existing contact.""" addresses = ListType(ModelType(PostalAddress), default=lambda: []) avatar = StringType(default='avatar.png') contact_id = UUIDType() date_insert = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') date_update = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') deleted = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') emails = ListType(ModelType(Email), default=lambda: []) identities = ListType(ModelType(SocialIdentity), default=lambda: []) ims = ListType(ModelType(IM), default=lambda: []) organizations = ListType(ModelType(Organization), default=lambda: []) phones = ListType(ModelType(Phone), default=lambda: []) pi = ModelType(PIParameter) user_id = UUIDType() class Options: serialize_when_none = False class ShortContact(Model): """Input structure for contact in short form.""" contact_id = UUIDType() family_name = StringType() given_name = StringType() tags = ListType(StringType(), default=lambda: []) title = StringType() pi = ModelType(PIParameter) class Options: serialize_when_none = False ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/parsers/__init__.py ================================================ # -*- coding: utf-8 -*- """Caliopen contact format parsers.""" from .vcard import VcardContact, VcardParser __all__ = ['VcardParser', 'VcardContact'] ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/parsers/vcard.py ================================================ """Caliopen vcard format parser.""" import logging import vobject import phonenumbers from caliopen_main.contact.parameters import NewContact, NewEmail, EMAIL_TYPES from caliopen_main.contact.parameters import NewIM, IM_TYPES, NewPhone from caliopen_main.contact.parameters import NewSocialIdentity from caliopen_main.contact.parameters import NewPostalAddress, NewOrganization log = logging.getLogger(__name__) class VcardContact(object): """Contact from a vcard entry.""" _meta = {} def __init__(self, vcard): """Parse a vcard contact.""" self._vcard = vcard self._parse() def _get_not_empty(self, prop): """Get non empty value (and only value) from a vcard property.""" if prop in self._vcard.contents: attr = self._vcard.contents[prop] if isinstance(attr, (list, tuple)): return [x.value for x in attr if x.value] return attr.value if attr.value else None return None def __build_email(self, param): email = NewEmail() email.address = param.value if 'TYPE' in param.params: email_type = param.params['TYPE'][0].lower() if email_type in EMAIL_TYPES: email.type = email_type if 'PREF' in param.params: email.is_primary = True return email def __parse_emails(self): """Read vcard email property and build NewEmail instances.""" for param in self._vcard.contents.get('email', []): yield self.__build_email(param) def __build_phone(self, param): # XXX TOFIX _vcard_types = { 'text': 'other', 'voice': 'other', 'fax': 'fax', 'cell': 'mobile', 'video': 'other', 'pager': 'pager', 'textphone': 'other', } phone = NewPhone() phone.number = param.value if 'TYPE' in param.params and param.params['TYPE']: phone_type = param.params['TYPE'][0].lower() if phone_type in _vcard_types: phone.type = _vcard_types[phone_type] else: phone.type = 'other' if 'PREF' in param.params: phone.is_primary = True try: number = phonenumbers.parse(phone.number, None) phone_format = phonenumbers.PhoneNumberFormat.INTERNATIONAL normalized = phonenumbers.format_number(number, phone_format) if normalized: phone.normalized_number = normalized except: pass return phone def __parse_phones(self): """Read vcard tel property and build NewPhone instances.""" for param in self._vcard.contents.get('tel', []): yield self.__build_phone(param) def __build_address(self, param): adr = NewPostalAddress() adr.city = param.value.city adr.zip = param.value.code adr.street = param.value.street adr.region = param.value.region adr.country = param.value.country return adr def __parse_addresses(self): """Read vcard adr property and build NewPostalAddress instances.""" for param in self._vcard.contents.get('adr', []): yield self.__build_address(param) def __build_organization(self, param): org = NewOrganization() org.name = param.value[0] return org def __parse_organizations(self): for param in self._vcard.contents.get('org', []): yield self.__build_organization(param) def __build_im(self, param): im = NewIM() im.address = param.value if 'TYPE' in param.params and param.params['TYPE']: im_type = param.params['TYPE'][0].lower() if im_type in IM_TYPES: im.type = im_type if 'PREF' in param.params: im.is_primary = True return im def __parse_impps(self): for param in self._vcard.contents.get('impp', []): yield self.__build_im(param) def __parse_social_identities(self): idents = [] if 'x-twitter' in self._vcard.contents: for ident in self._vcard.contents.get('x-twitter', []): social = NewSocialIdentity() social.type = 'twitter' social.name = ident.value idents.append(social) return idents def _parse(self): contact = NewContact() if 'n' in self._vcard.contents: contact.given_name = self._vcard.n.value.given contact.family_name = self._vcard.n.value.family contact.title = '{0} {1}'.format(contact.given_name, contact.family_name) elif 'fn' in self._vcard.contents: contact.title = self._vcard.fn.value elif 'cn' in self._vcard.contents: contact.title = self._vcard.contents['cn'][0].value elif self._vcard.contents.get('email'): contact.title = self._vcard.contents['email'][0].value if not contact.infos: contact.infos = {} for prop in ['nickname']: value = self._get_not_empty(prop) if value and len(value): contact.infos[prop] = value[0] for prop in ['uid', 'rev']: value = self._get_not_empty(prop) if value: self._meta[prop] = value contact.phones = self.__parse_phones() contact.emails = self.__parse_emails() contact.addresses = self.__parse_addresses() contact.organizations = self.__parse_organizations() contact.ims = self.__parse_impps() contact.identities = self.__parse_social_identities() self.contact = contact def serialize(self): """Serialize contact.""" data = self.contact.serialize() data.update({'_meta': self._meta}) return data def validate(self): """Validate contact parsed informations.""" return self.contact.validate() class VcardParser(object): """Vcard format parser class.""" def __init__(self, f): """Read a vcard file and create a generator on vcard objects.""" self._vcards = vobject.readComponents(f) def parse(self): """Generator on vcards objects read from read file.""" for vcard in self._vcards: yield VcardContact(vcard) ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/returns.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from caliopen_storage.parameters import ReturnCoreObject from .core import Contact from .parameters import (Contact as ContactParam, ShortContact as ContactShortParam, Email as EmailParam, Phone as PhoneParam, IM as IMParam, Organization as OrganizationParam, PostalAddress as PostalAddressParam, SocialIdentity as SocialIdentityParam) class ReturnContact(ReturnCoreObject): _core_class = Contact _return_class = ContactParam class ReturnShortContact(ReturnCoreObject): _core_class = Contact _return_class = ContactShortParam class ReturnEmail(ReturnCoreObject): _return_class = EmailParam class ReturnIM(ReturnCoreObject): _return_class = IMParam class ReturnPhone(ReturnCoreObject): _return_class = PhoneParam class ReturnAddress(ReturnCoreObject): _return_class = PostalAddressParam class ReturnSocialIdentity(ReturnCoreObject): _return_class = SocialIdentityParam class ReturnOrganization(ReturnCoreObject): _return_class = OrganizationParam ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/store/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .contact import Contact, IndexedContact, ContactLookup from .contact import Organization, PostalAddress from .contact import Email, IM, Phone, SocialIdentity __all__ = ['Contact', 'ContactLookup', 'IndexedContact', 'Organization', 'PostalAddress', 'Email', 'IM', 'Phone', 'SocialIdentity'] ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/store/contact.py ================================================ # -*- coding: utf-8 -*- """Caliopen objects related to contact definition.""" from __future__ import absolute_import, print_function, unicode_literals import uuid from cassandra.cqlengine import columns from caliopen_storage.store.mixin import IndexedModelMixin from caliopen_storage.store import BaseModel, BaseUserType from caliopen_main.pi.objects import PIModel from .contact_index import IndexedContact class Organization(BaseUserType): """Contact organizations model.""" _pkey = 'organization_id' deleted = columns.Boolean(default=False) department = columns.Text() is_primary = columns.Boolean(default=False) job_description = columns.Text() label = columns.Text() name = columns.Text() organization_id = columns.UUID(default=uuid.uuid4) title = columns.Text() type = columns.Text() # work, other class PostalAddress(BaseUserType): """Contact postal addresses model.""" _pkey = 'address_id' address_id = columns.UUID(default=uuid.uuid4) city = columns.Text() country = columns.Text() is_primary = columns.Boolean(default=False) label = columns.Text() postal_code = columns.Text() region = columns.Text() street = columns.Text() type = columns.Text() class Email(BaseUserType): """Contact emails model.""" _pkey = 'email_id' uniq_name = 'address' address = columns.Text() email_id = columns.UUID(default=uuid.uuid4) is_primary = columns.Boolean(default=False) label = columns.Text() type = columns.Text() # home, work, other class IM(BaseUserType): """Contact instant messaging adresses model.""" _pkey = 'im_id' uniq_name = 'address' address = columns.Text() im_id = columns.UUID(default=uuid.uuid4) is_primary = columns.Boolean(default=False) label = columns.Text() protocol = columns.Text() type = columns.Text() class Phone(BaseUserType): """Contact phones model.""" _pkey = 'phone_id' uniq_name = 'number' is_primary = columns.Boolean(default=False) number = columns.Text() normalized_number = columns.Text() phone_id = columns.UUID(default=uuid.uuid4) type = columns.Text() uri = columns.Text() # RFC3966 class SocialIdentity(BaseUserType): """Any contact social identity (facebook, twitter, linkedin, etc).""" _pkey = 'social_id' social_id = columns.UUID(default=uuid.uuid4) name = columns.Text() type = columns.Text() # Abstract everything else in a map infos = columns.Map(columns.Text, columns.Text) class Contact(BaseModel, IndexedModelMixin): """Contact model.""" _index_class = IndexedContact user_id = columns.UUID(primary_key=True) contact_id = columns.UUID(primary_key=True) # clustering key additional_name = columns.Text() addresses = columns.List(columns.UserDefinedType(PostalAddress)) avatar = columns.Text() date_insert = columns.DateTime() date_update = columns.DateTime() deleted = columns.DateTime() emails = columns.List(columns.UserDefinedType(Email)) family_name = columns.Text() given_name = columns.Text() groups = columns.List(columns.Text()) identities = columns.List(columns.UserDefinedType(SocialIdentity)) ims = columns.List(columns.UserDefinedType(IM)) infos = columns.Map(columns.Text, columns.Text) name_prefix = columns.Text() name_suffix = columns.Text() organizations = columns.List(columns.UserDefinedType(Organization)) phones = columns.List(columns.UserDefinedType(Phone)) pi = columns.UserDefinedType(PIModel) privacy_features = columns.Map(columns.Text(), columns.Text()) tags = columns.List(columns.Text(), db_field="tagnames") title = columns.Text() # computed value, read only class ContactLookup(BaseModel): """Lookup any information needed to recognize a user contact.""" user_id = columns.UUID(primary_key=True) value = columns.Text( primary_key=True) # address or 'identifier' in identity type = columns.Text(primary_key=True) # email, IM, etc. contact_id = columns.UUID() ================================================ FILE: src/backend/main/py.main/caliopen_main/contact/store/contact_index.py ================================================ # -*- coding: utf-8 -*- """Caliopen contact index classes.""" from __future__ import absolute_import, print_function, unicode_literals import logging from elasticsearch_dsl import Mapping, Nested, Text, Keyword, Date, Boolean, \ InnerObjectWrapper, Object from caliopen_storage.store.model import BaseIndexDocument from caliopen_main.pi.objects import PIIndexModel log = logging.getLogger(__name__) class IndexedOrganization(InnerObjectWrapper): """Contact indexed organization model.""" deleted = Boolean() department = Text() is_primary = Boolean() job_description = Text() label = Text() name = Keyword() organization_id = Keyword() title = Keyword() type = Keyword() class IndexedPostalAddress(InnerObjectWrapper): """Contact indexed postal addresse model.""" address_id = Keyword() label = Text() type = Keyword() is_primary = Boolean() street = Text() city = Text() postal_code = Text() country = Text() region = Text() class IndexedInternetAddress(InnerObjectWrapper): """Contact indexed address on internet (email, im) model.""" address = Keyword() email_id = Keyword() is_primary = Boolean() label = Text() type = Keyword() class IndexedPhone(InnerObjectWrapper): """Contact indexed phone model.""" is_primary = Boolean() number = Text() normalized_number = Text() phone_id = Keyword() type = Keyword() uri = Keyword() class IndexedSocialIdentity(InnerObjectWrapper): """Contact indexed social identity model.""" name = Text() type = Keyword() # Abstract everything else in a map infos = Nested() class IndexedContact(BaseIndexDocument): """Indexed contact model.""" doc_type = 'indexed_contact' user_id = Keyword() contact_id = Keyword() additional_name = Keyword() addresses = Nested(doc_class=IndexedPostalAddress) avatar = Keyword() date_insert = Date() date_update = Date() deleted = Date() emails = Nested(doc_class=IndexedInternetAddress) family_name = Keyword() given_name = Keyword() groups = Keyword(multi=True) identities = Nested(doc_class=IndexedSocialIdentity) ims = Nested(doc_class=IndexedInternetAddress) infos = Nested() name_prefix = Keyword() name_suffix = Keyword() organizations = Nested(doc_class=IndexedOrganization) phones = Nested(doc_class=IndexedPhone) pi = Object(doc_class=PIIndexModel) privacy_features = Object() public_key = Nested() social_identities = Nested(doc_class=IndexedSocialIdentity) tags = Keyword(multi=True) title = Text() @property def contact_id(self): """The compound primary key for a contact is contact_id.""" return self.meta.id @classmethod def build_mapping(cls): """Create elasticsearch indexed_contacts mapping object for an user.""" m = Mapping(cls.doc_type) m.meta('_all', enabled=True) m.field('user_id', 'keyword') m.field('contact_id', 'keyword') m.field('additional_name', 'text', fields={ "normalized": {"type": "text", "analyzer": "text_analyzer"} }) # addresses addresses = Nested(doc_class=IndexedPostalAddress, include_in_all=True, properties={ "address_id": "keyword", "label": "text", "type": "keyword", "is_primary": "boolean", "street": "text", "city": "text", "postal_code": "keyword", "country": "text", "region": "text" }) m.field("addresses", addresses) m.field("avatar", "keyword") m.field('date_insert', 'date') m.field('date_update', 'date') m.field('deleted', 'date') # emails internet_addr = Nested(doc_class=IndexedInternetAddress, include_in_all=True, ) internet_addr.field("address", "text", analyzer="text_analyzer", fields={ "raw": {"type": "keyword"}, "parts": {"type": "text", "analyzer": "email_analyzer"} }) internet_addr.field("email_id", Keyword()) internet_addr.field("is_primary", Boolean()) internet_addr.field("label", "text", analyzer="text_analyzer") internet_addr.field("type", Keyword()) m.field("emails", internet_addr) m.field('family_name', "text", fields={ "normalized": {"type": "text", "analyzer": "text_analyzer"} }) m.field('given_name', 'text', fields={ "normalized": {"type": "text", "analyzer": "text_analyzer"} }) m.field("groups", Keyword(multi=True)) # social ids social_ids = Nested(doc_class=IndexedSocialIdentity, include_in_all=True, properties={ "name": "text", "type": "keyword", "infos": Nested() }) m.field("identities", social_ids) m.field("ims", internet_addr) m.field("infos", Nested()) m.field('name_prefix', 'keyword') m.field('name_suffix', 'keyword') # organizations organizations = Nested(doc_class=IndexedOrganization, include_in_all=True) organizations.field("deleted", Boolean()) organizations.field("department", "text", analyzer="text_analyzer") organizations.field("is_primary", Boolean()) organizations.field("job_description", "text") organizations.field("label", "text", analyzer="text_analyzer") organizations.field("name", 'text', fields={ "normalized": {"type": "text", "analyzer": "text_analyzer"} }) organizations.field("organization_id", Keyword()) organizations.field("title", Keyword()) organizations.field("type", Keyword()) m.field("organizations", organizations) # phones phones = Nested(doc_class=IndexedPhone, include_in_all=True, properties={ "is_primary": "boolean", "number": "text", "normalized_number": "text", "phone_id": "keyword", "type": "keyword", "uri": "keyword" }) m.field("phones", phones) # pi pi = Object(doc_class=PIIndexModel, include_in_all=True, properties={ "comportment": "integer", "context": "integer", "date_update": "date", "technic": "integer", "version": "integer" }) m.field("pi", pi) m.field("privacy_features", Object(include_in_all=True)) m.field("public_key", Nested()) m.field("social_identities", social_ids) m.field("tags", Keyword(multi=True)) m.field('title', 'text', analyzer="text_analyzer", fields={ "raw": {"type": "keyword"} }) return m ================================================ FILE: src/backend/main/py.main/caliopen_main/device/__init__.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/device/core.py ================================================ # -*- coding: utf-8 -*- """Caliopen device core classes.""" from __future__ import absolute_import, print_function, unicode_literals import logging import uuid from .store import Device as ModelDevice from .store import DeviceLocation as ModelDeviceLocation from .store import DeviceConnectionLog as ModelDeviceLog from caliopen_storage.core.mixin import MixinCoreRelation, MixinCoreNested from caliopen_main.common.core import BaseUserCore from caliopen_main.common.core import PublicKey, BaseUserRelatedCore from caliopen_main.common.parameters import NewPublicKey from caliopen_main.device.parameters import NewDevice from caliopen_pi.qualifiers import NewDeviceQualifier log = logging.getLogger(__name__) class DeviceLocation(BaseUserRelatedCore): """Locations defined for a device to restrict access.""" _model_class = ModelDeviceLocation _pkey_name = 'address' class DeviceLog(BaseUserRelatedCore): """Locations defined for a device to restrict access.""" _model_class = ModelDeviceLog _pkey_name = 'timestamp' class Device(BaseUserCore, MixinCoreRelation, MixinCoreNested): """User device core class.""" _model_class = ModelDevice _pkey_name = 'device_id' _relations = { 'public_keys': PublicKey, 'locations': DeviceLocation, } @classmethod def create_from_parameter(cls, user, param, headers): """Create a device from API parameters.""" dev = NewDevice() dev.device_id = param['device_id'] dev.user_agent = headers.get('User-Agent') dev.ip_creation = headers.get('X-Forwarded-For') dev.status = 'verified' if 'status' not in param else param['status'] qualifier = NewDeviceQualifier(user) qualifier.process(dev) if 'name' in param: # Used to set the default device dev.name = param['name'] else: dev_name = 'new device' if 'device_type' in dev.privacy_features: dev_ext = dev.privacy_features['device_type'] dev_name = '{} {}'.format(dev_name, dev_ext) dev.name = dev_name dev.type = dev.privacy_features.get('device_type', 'other') # Device ecdsa key dev_key = NewPublicKey() dev_key.key_id = uuid.uuid4() dev_key.resource_id = dev.device_id dev_key.resource_type = 'device' dev_key.label = 'ecdsa key' dev_key.kty = 'ec' dev_key.use = 'sig' dev_key.x = int(param['ecdsa_key']['x'], 16) dev_key.y = int(param['ecdsa_key']['y'], 16) dev_key.crv = param['ecdsa_key']['curve'] # XXX Should be better design alg_map = { 'P-256': 'ES256', 'P-384': 'ES384', 'P-521': 'ES512' } dev_key.alg = alg_map[dev_key.crv] return Device.create(user, dev, public_keys=[dev_key]) @classmethod def create(cls, user, device, **related): """Create a new device for an user.""" device.validate() attrs = {'device_id': device.device_id, 'type': device.type, 'name': device.name, 'user_agent': device.user_agent, 'ip_creation': device.ip_creation, 'privacy_features': device.privacy_features, 'status': device.status} core = super(Device, cls).create(user, **attrs) log.debug('Created device %s' % core.device_id) related_cores = {} for k, v in related.iteritems(): if k in cls._relations: for obj in v: log.debug('Processing object %r' % obj.to_native()) # XXX check only one is_primary per relation using it new_core = core._relations[k].create(user, **obj) related_cores.setdefault(k, []).append(new_core.to_dict()) log.debug('Created related core %r' % new_core) return core def check_locations(self, ipaddr): """Check if the location of device is in related authorized ips.""" if self.locations: for loc in self.locations: log.info("Testing cidr {0} with ip {1}". format(loc.address, ipaddr)) return True def _log_action(self, ipaddr, action): DeviceLog.create(self.user, self.device_id, ipaddr=ipaddr, type=action) def login(self, ipaddr): """Login action for a device.""" log.info("Logging device {0} for user {1}".format(self.device_id, self.user_id)) if not self.check_locations(ipaddr): raise Exception("Device IP Address not in allowed locations") self._log_action(ipaddr, 'login') def logout(self, ipaddr): """Logout action for a device.""" self._log_action(ipaddr, "logout") ================================================ FILE: src/backend/main/py.main/caliopen_main/device/parameters.py ================================================ # -*- coding: utf-8 -*- """Caliopen device parameters classes.""" from __future__ import absolute_import, print_function, unicode_literals from schematics.models import Model from schematics.transforms import blacklist from schematics.types import DateTimeType, StringType, UUIDType from schematics.types.compound import ListType, ModelType, DictType from caliopen_main.pi.parameters import PIParameter import caliopen_storage.helpers.json as helpers DEVICE_TYPES = ['other', 'desktop', 'laptop', 'smartphone', 'tablet'] class DeviceLocation(Model): """Location structure for a device.""" user_id = UUIDType() device_id = UUIDType() address = StringType(required=True) # With CIDR notation type = StringType() country = StringType() class Options: serialize_when_none = False roles = {'default': blacklist('user_id', 'device_id')} class NewDevice(Model): """Structure to create a new user device.""" name = StringType(required=True) type = StringType(required=True, choices=DEVICE_TYPES, default='other') locations = ListType(ModelType(DeviceLocation), default=lambda: []) user_agent = StringType() ip_creation = StringType() class Device(NewDevice): """Parameter for an existing device.""" device_id = UUIDType() user_id = UUIDType() date_insert = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') date_revoked = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') status = StringType() privacy_features = DictType(StringType, default=lambda: {}) pi = ModelType(PIParameter) class Options: serialize_when_none = False roles = {'default': blacklist('user_id')} ================================================ FILE: src/backend/main/py.main/caliopen_main/device/store.py ================================================ # -*- coding: utf-8 -*- """Caliopen cassandra models related to device.""" from __future__ import absolute_import, print_function, unicode_literals import logging import uuid from datetime import datetime from cassandra.cqlengine import columns from caliopen_storage.store.model import BaseModel from caliopen_main.pi.objects import PIModel log = logging.getLogger(__name__) class DeviceLocation(BaseModel): """Device defined location, based on IP address.""" user_id = columns.UUID(primary_key=True) device_id = columns.UUID(primary_key=True) address = columns.Text(primary_key=True) # IP address with CIDR type = columns.Text() # home/work/etc country = columns.Text() class Device(BaseModel): """User device.""" user_id = columns.UUID(primary_key=True) device_id = columns.UUID(primary_key=True, default=uuid.uuid4) name = columns.Text() date_insert = columns.DateTime(required=True, default=datetime.utcnow) date_revoked = columns.DateTime() type = columns.Text(required=True) # laptop, desktop, smartphone, etc status = columns.Text(default='unverified') user_agent = columns.Text() ip_creation = columns.Text() privacy_features = columns.Map(columns.Text, columns.Text) pi = columns.UserDefinedType(PIModel) class DeviceConnectionLog(BaseModel): """Log a device connection.""" user_id = columns.UUID(primary_key=True) resource_id = columns.UUID(primary_key=True) # device_id date_insert = columns.DateTime(primary_key=True, default=datetime.utcnow) ip_address = columns.Text() type = columns.Text() # Connection type (login/logout) country = columns.Text() # Geoip detected country ================================================ FILE: src/backend/main/py.main/caliopen_main/discussion/__init__.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/discussion/core/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .discussion import Discussion __all__ = ['Discussion'] ================================================ FILE: src/backend/main/py.main/caliopen_main/discussion/core/discussion.py ================================================ # -*- coding: utf-8 -*- """Caliopen core discussion related classes.""" from __future__ import absolute_import, print_function, unicode_literals import logging from caliopen_main.common.objects.base import ObjectUser from caliopen_main.participant.core import participants_from_uris, \ hash_participants_uri from caliopen_main.participant.core import ParticipantHash log = logging.getLogger(__name__) class Discussion(ObjectUser): """Discussion core object.""" def upsert_lookups_for_participants(self, participants): """ Ensure that participants lookup and hash lookup tables are filled for these participants :param user: :param participants: a collection of parameters/Participant :type uris: set :return: Discussion """ if not participants: raise Exception("missing mandatory property to create lookup entry") uris = hash_participants_uri(participants) hash_lookup = ParticipantHash.find(user_id=self.user.user_id, kind="uris", key=uris['hash']) if len(hash_lookup) > 0: self.participants = hash_lookup[0].components self.participants_hash = hash_lookup[0].value else: parts = participants_from_uris(self.user, uris['uris'], uris['hash']) self.participants = parts['components'] self.participants_hash = parts['hash'] return self ================================================ FILE: src/backend/main/py.main/caliopen_main/discussion/objects/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .discussion import Discussion __all__ = {'Discussion'} ================================================ FILE: src/backend/main/py.main/caliopen_main/discussion/objects/discussion.py ================================================ # -*- coding: utf-8 -*- """Caliopen message object classes.""" from __future__ import absolute_import, print_function, unicode_literals import types from caliopen_main.common.objects.base import ObjectUser from caliopen_main.participant.core import hash_participants_uri, \ participants_from_uris from caliopen_main.participant.store import ParticipantHash class Discussion(ObjectUser): _attrs = { 'uris_hash': types.StringType, 'uris': [types.StringType], 'participants_hash': types.StringType, 'participants': [types.StringType], } def upsert_lookups_for_participants(self, user, participants): """ Ensure that participants lookup and hash lookup tables are filled for these participants :param user: :param participants: a collection of parameters/Participant :type uris: set :return: Discussion """ if not participants: raise Exception("missing mandatory property to create lookup entry") uris = hash_participants_uri(participants) self.uris = uris['uris'] self.uris_hash = uris['hash'] hash_lookup = ParticipantHash.find(user_id=user.user_id, kind="uris", key=uris['hash']) if len(hash_lookup) > 0: self.participants = hash_lookup[0].components self.participants_hash = hash_lookup[0].value else: parts = participants_from_uris(user, uris['uris'], uris['hash']) self.participants = parts['components'] self.participants_hash = parts['hash'] return self ================================================ FILE: src/backend/main/py.main/caliopen_main/discussion/parameters/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .discussion import Discussion __all__ = [ 'Discussion' ] ================================================ FILE: src/backend/main/py.main/caliopen_main/discussion/parameters/discussion.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from schematics.models import Model from schematics.types import (StringType, DateTimeType, IntType, UUIDType, BooleanType) from schematics.types.compound import ListType, ModelType from schematics.transforms import blacklist from caliopen_main.participant.parameters import Participant import caliopen_storage.helpers.json as helpers class Discussion(Model): """Existing discussion.""" user_id = UUIDType() discussion_id = StringType(required=True) # = participants_hash date_insert = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') date_update = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') last_message_subject = StringType() excerpt = StringType(required=True) importance_level = IntType(required=True, default=0) participants = ListType(ModelType(Participant), default=lambda: []) total_count = IntType(required=True, default=0) unread_count = IntType(required=True, default=0) attachment_count = IntType(default=0) subject = StringType() protocol = StringType() last_message_id = UUIDType(required=True) class Options: roles = {'default': blacklist('user_id')} serialize_when_none = False ================================================ FILE: src/backend/main/py.main/caliopen_main/discussion/store/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .discussion_index import DiscussionIndexManager __all__ = ['DiscussionIndexManager'] ================================================ FILE: src/backend/main/py.main/caliopen_main/discussion/store/discussion_index.py ================================================ # -*- coding: utf-8 -*- """Caliopen disccions index classes. Discussions are not really indexed, they are result of messages aggregations. So there is not direct document mapping, only helpers to find discussions and build a suitable representation for displaying. """ from __future__ import absolute_import, print_function, unicode_literals import logging from elasticsearch_dsl import A from caliopen_storage.store.model import BaseIndexDocument from caliopen_main.message.store.message_index import IndexedMessage log = logging.getLogger(__name__) class DiscussionIndex(object): """Informations from index about a discussion.""" total_count = 0 unread_count = 0 attachment_count = 0 last_message = None def __init__(self, id): self.discussion_id = id class DiscussionIndexManager(object): """Manager for building discussions from index storage layer.""" def __init__(self, user): self.index = user.shard_id self.user_id = user.user_id self.proxy = BaseIndexDocument.client() def _prepare_search(self): """Prepare a dsl.Search object on current index.""" search = IndexedMessage.search(using=self.proxy, index=self.index) search = search.filter('term', user_id=self.user_id) return search def __search_ids(self, limit, offset, min_pi, max_pi, min_il, max_il): """Search discussions ids as a bucket aggregation.""" # TODO : search on participants_hash instead search = self._prepare_search(). \ filter("range", importance_level={'gte': min_il, 'lte': max_il}) # Do bucket term aggregation, sorted by last_message date size = offset + (limit * 2) agg = A('terms', field='discussion_id', order={'last_message': 'desc'}, size=size, shard_size=size) search.aggs.bucket('discussions', agg) \ .metric('last_message', 'max', field='date_sort') \ .bucket("unread", "filter", term={"is_unread": True}) result = search.source(exclude=["*"]).execute() if hasattr(result, 'aggregations'): # Something found buckets = result.aggregations.discussions.buckets # XXX Ugly but don't find a way to paginate on bucket aggregation buckets = buckets[offset:offset + limit] total = result.aggregations.discussions.sum_other_doc_count # remove last_message for now as it doesn't have relevant attrs for discussion in buckets: del discussion["last_message"] return buckets, total log.debug('No result found on index {}'.format(self.index)) return {}, 0 def get_last_message(self, discussion_id, min_il, max_il, include_draft): """Get last message of a given discussion.""" search = self._prepare_search() \ .filter("match", discussion_id=discussion_id) \ .filter("range", importance_level={'gte': min_il, 'lte': max_il}) if not include_draft: search = search.filter("match", is_draft=False) result = search.sort('-date_sort')[0:1].execute() if not result.hits: # XXX what to do better if not found ? return {} return result.hits[0] def list_discussions(self, limit=10, offset=0, min_pi=0, max_pi=0, min_il=-10, max_il=10): """Build a list of limited number of discussions.""" buckets, total = self.__search_ids(limit, offset, min_pi, max_pi, min_il, max_il) discussions = [] for bucket in buckets: # TODO : les buckets seront des hash_participants, donc il faut créer la liste des discussion_id avant et itérer là-dessus message = self.get_last_message(bucket['key'], min_il, max_il, True) discussion = DiscussionIndex(bucket['key']) discussion.total_count = bucket['doc_count'] discussion.unread_count = bucket['unread']['doc_count'] discussion.last_message = message # XXX build others values from index discussions.append(discussion) # XXX total do not work completly, hack a bit return discussions, total + len(discussions) def message_belongs_to(self, discussion_id, message_id): """Search if a message belongs to a discussion""" msg = IndexedMessage.get(message_id, using=self.proxy, index=self.index) return str(msg.discussion_id) == str(discussion_id) def get_by_id(self, discussion_id, min_il=0, max_il=100): """Return a single discussion by discussion_id""" # TODO : search by multiple discussion_id because they are hashes now search = self._prepare_search() \ .filter("match", discussion_id=discussion_id) search.aggs.bucket('discussions', A('terms', field='discussion_id')) \ .bucket("unread", "filter", term={"is_unread": True}) result = search.execute() if not result.hits or len(result.hits) < 1: return None message = self.get_last_message(discussion_id, min_il, max_il, True) discussion = DiscussionIndex(discussion_id) discussion.total_count = result.hits.total discussion.last_message = message discussion.unread_count = result.aggregations.discussions.buckets[ 0].unread.doc_count return discussion def get_by_uris(self, uris_hashes, min_il=0, max_il=100): """ :param uris_hashes: an array of uris hashes :param min_il: :param max_il: :return: """ search = self._prepare_search(). \ filter("terms", discussion_id=uris_hashes). \ filter("range", importance_level={'gte': min_il, 'lte': max_il}) agg = A('terms', field='discussion_id', order={'last_message': 'desc'}) search.aggs.bucket('discussions', agg). \ metric('last_message', 'max', field='date_sort'). \ bucket("unread", "filter", term={"is_unread": True}) result = search.execute() if not result.hits or len(result.hits) < 1: return None return result ================================================ FILE: src/backend/main/py.main/caliopen_main/discussion/store/discussion_lookup.py ================================================ # -*- coding: utf-8 -*- """Caliopen storage model for messages.""" from __future__ import absolute_import, print_function, unicode_literals from cassandra.cqlengine import columns from caliopen_storage.store.model import BaseModel # TODO : make user of ParticipantLookup table class DiscussionListLookup(BaseModel): """Lookup discussion by external list-id.""" user_id = columns.UUID(primary_key=True) list_id = columns.Text(primary_key=True) discussion_id = columns.UUID() # TODO : primary ? class DiscussionThreadLookup(BaseModel): """Lookup discussion by external thread's root message_id.""" user_id = columns.UUID(primary_key=True) external_root_msg_id = columns.Text(primary_key=True) discussion_id = columns.UUID() class DiscussionLookup(BaseModel): user_id = columns.UUID(primary_key=True) key = columns.Text(primary_key=True) value = columns.Text(primary_key=True) date_insert = columns.DateTime() ================================================ FILE: src/backend/main/py.main/caliopen_main/message/__init__.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/message/core/__init__.py ================================================ from .raw import RawMessage, UserRawLookup from .external_references import MessageExternalRefLookup __all__ = ['RawMessage', 'UserRawLookup', 'MessageExternalRefLookup'] ================================================ FILE: src/backend/main/py.main/caliopen_main/message/core/external_references.py ================================================ # -*- coding: utf-8 -*- """Caliopen core message external references lookup class.""" from __future__ import absolute_import, print_function, unicode_literals from caliopen_main.common.core import BaseUserCore from ..store import MessageExternalRefLookup as ModelMessageExternalRefLookup class MessageExternalRefLookup(BaseUserCore): """Lookup message by external message-id""" _model_class = ModelMessageExternalRefLookup _pkey_name = 'external_msg_id' ================================================ FILE: src/backend/main/py.main/caliopen_main/message/core/raw.py ================================================ # -*- coding: utf-8 -*- """Caliopen core raw message class.""" from __future__ import absolute_import, print_function, unicode_literals import uuid import logging from minio import Minio from minio.error import ResponseError import urlparse from minio import Minio from minio.error import ResponseError from caliopen_storage.core import BaseCore from caliopen_main.common.core import BaseUserCore from caliopen_storage.exception import NotFound from caliopen_storage.config import Configuration from ..store import (RawMessage as ModelRaw, UserRawLookup as ModelUserRawLookup) from caliopen_main.message.parsers.mail import MailMessage log = logging.getLogger(__name__) class RawMessage(BaseCore): """ Raw message core. Store in binary form any message before processing """ _model_class = ModelRaw _pkey_name = 'raw_msg_id' @classmethod def create(cls, raw): """Create raw message.""" key = uuid.uuid4() size = len(raw) return super(RawMessage, cls).create(raw_msg_id=key, raw_data=raw, raw_size=size) @classmethod def get(cls, raw_msg_id): """ Get raw message from db or ObjectStorage service :param raw_msg_id: :return: a RawMessage or NotFound exception """ try: raw_msg = super(RawMessage, cls).get(raw_msg_id) except Exception as exc: log.warn(exc) raise NotFound if len(raw_msg.raw_data) == 0 and raw_msg.uri != '': # means raw message data have been stored in object store # need to retrieve raw_data from it url = urlparse.urlsplit(raw_msg.uri) path = url.path.strip("/") if url.scheme == 's3': minioConf = Configuration("global").get("object_store") minioClient = Minio(minioConf["endpoint"], access_key=minioConf["access_key"], secret_key=minioConf["secret_key"], secure=False, region=minioConf["location"]) try: resp = minioClient.get_object(url.netloc, path) except Exception as exc: log.warn(exc) raise NotFound # resp is a urllib3.response.HTTPResponse class try: raw_msg.raw_data = resp.data except Exception as exc: log.warn(exc) raise NotFound else: log.warn("raw message uri scheme not implemented") raise NotFound return raw_msg @classmethod def get_for_user(cls, user_id, raw_msg_id): """ Get raw message by raw_msg_id, if message belongs to user. :param: user_id is a string :param: raw_msg_id is a string :return: a RawMessage or None """ if not UserRawLookup.belongs_to_user(user_id, raw_msg_id): return None try: return cls.get(raw_msg_id) except NotFound: return None def parse(self): """Parse raw message to get a formatted object.""" return MailMessage(self) class UserRawLookup(BaseUserCore): """User raw message affectation.""" _model_class = ModelUserRawLookup _pkey_name = 'raw_msg_id' ================================================ FILE: src/backend/main/py.main/caliopen_main/message/objects/__init__.py ================================================ from .message import Message __all__ = ['Message'] ================================================ FILE: src/backend/main/py.main/caliopen_main/message/objects/attachment.py ================================================ # -*- coding: utf-8 -*- """Caliopen message object classes.""" from __future__ import absolute_import, print_function, unicode_literals import types from uuid import UUID from caliopen_main.common.objects.base import ObjectJsonDictifiable from ..store.attachment import MessageAttachment as ModelMessageAttachment from ..store.attachment_index import IndexedMessageAttachment class MessageAttachment(ObjectJsonDictifiable): """attachment's attributes, nested within message object""" _attrs = { 'content_type': types.StringType, 'file_name': types.SliceType, 'is_inline': types.BooleanType, 'size': types.IntType, 'temp_id': UUID, 'url': types.StringType, 'mime_boundary': types.StringType } _model_class = ModelMessageAttachment _index_class = IndexedMessageAttachment ================================================ FILE: src/backend/main/py.main/caliopen_main/message/objects/external_references.py ================================================ # -*- coding: utf-8 -*- """Caliopen message object classes.""" from __future__ import absolute_import, print_function, unicode_literals import types from caliopen_main.common.objects.base import ObjectJsonDictifiable from ..store.external_references import ExternalReferences as ModelExtRef from ..store.external_references_index import IndexedExternalReferences class ExternalReferences(ObjectJsonDictifiable): """external references, nested within message object""" _attrs = { 'ancestors_ids': [types.StringType], 'message_id': types.StringType, 'parent_id': types.StringType } _model_class = ModelExtRef _index_class = IndexedExternalReferences ================================================ FILE: src/backend/main/py.main/caliopen_main/message/objects/message.py ================================================ # -*- coding: utf-8 -*- """Caliopen message object classes.""" from __future__ import absolute_import, print_function, unicode_literals import types from caliopen_main.common.objects.base import ObjectIndexable import uuid from uuid import UUID import datetime import pytz import json import copy from caliopen_storage.config import Configuration from caliopen_main.pi.objects import PIObject from ..store import Message as ModelMessage from ..store import IndexedMessage from ..parameters.message import Message as ParamMessage from ..parameters.draft import Draft from ..core import RawMessage from .attachment import MessageAttachment from .external_references import ExternalReferences from caliopen_main.participant.objects.participant import Participant from schematics.types import UUIDType from caliopen_main.participant.parameters import \ Participant as IndexedParticipant from caliopen_main.common import errors as err import logging log = logging.getLogger(__name__) class Message(ObjectIndexable): """Message object class.""" # TODO : manage attrs that should not be editable directly by users _attrs = { 'attachments': [MessageAttachment], 'body_html': types.StringType, 'body_plain': types.StringType, 'date': datetime.datetime, 'date_delete': datetime.datetime, 'date_insert': datetime.datetime, 'date_sort': datetime.datetime, 'discussion_id': types.StringType, 'external_references': ExternalReferences, 'importance_level': types.IntType, 'is_answered': types.BooleanType, 'is_draft': types.BooleanType, 'is_unread': types.BooleanType, 'is_received': types.BooleanType, 'message_id': UUID, 'parent_id': UUID, 'participants': [Participant], 'privacy_features': types.DictType, 'pi': PIObject, 'raw_msg_id': UUID, 'subject': types.StringType, 'tags': [types.StringType], 'protocol': types.StringType, 'user_id': UUID, 'user_identities': [UUID], } _json_model = ParamMessage # operations related to cassandra _model_class = ModelMessage _db = None # model instance with datas from db _pkey_name = "message_id" # operations related to elasticsearch _index_class = IndexedMessage _index = None @property def raw(self): """Return raw text from pristine raw message.""" msg = RawMessage.get_for_user(self.user_id, self.raw_msg_id) return msg.raw_data @property def raw_json(self): """Return json representation of pristine raw message.""" msg = RawMessage.get_for_user(self.user_id, self.raw_msg_id) return json.loads(msg.json_rep) @property def external_msg_id(self): if self.external_references: return self.external_references.message_id return None @property def user_identity(self): """ return first user_identity """ return self.user_identities[0] if self.user_identities else None @classmethod def create_draft(cls, user, **params): """ Create and save a new message (draft) for an user. :params: a NewMessage dict """ # silently remove unexpected props within patch if not in strict mode strict_patch = Configuration('global').get('apiV1.strict_patch', False) if not strict_patch: allowed_properties = [ "body", "message_id", "parent_id", "participants", "subject", "user_identities", "privacy_features", ] for key, value in params.items(): if key not in allowed_properties: del (params[key]) try: draft_param = Draft(params, strict=strict_patch) if draft_param.message_id: draft_param.validate_uuid(user.user_id) else: draft_param.message_id = uuid.uuid4() discussion_id = draft_param.validate_consistency(user, True) except Exception as exc: log.warn("create_draft error %r" % exc) raise exc message = Message(user) message.unmarshall_json_dict(draft_param.to_primitive()) message.user_id = UUID(user.user_id) message.is_draft = True message.is_received = False message.discussion_id = discussion_id if not message.protocol: log.warn("failed to pick a protocol") raise Exception("`message protocol is missing") # forbid multiple protocol for participant in message.participants: if participant.protocol != message.protocol: log.warning("Different protocols detected {0} and {1}". format(participant.protocol, message.protocol)) message.date = message.date_sort = message.date_insert = \ datetime.datetime.now(tz=pytz.utc) try: message.marshall_db() message.save_db() except Exception as exc: log.warn(exc) raise exc try: message.marshall_index() message.save_index(wait_for=True) except Exception as exc: log.warn(exc) raise exc return message def patch_draft(self, user, patch, **options): """Operation specific to draft, before applying generic patch.""" try: params = dict(patch) except Exception as exc: log.info(exc) raise err.PatchError(message=exc.message) # silently remove unexpected props within patch if not in strict mode strict_patch = Configuration('global').get('apiV1.strict_patch', False) if not strict_patch: allowed_properties = [ "body", "current_state", "user_identities", "message_id", "parent_id", "participants", "subject", "privacy_features", ] for key, value in params.items(): if key not in allowed_properties: del (params[key]) for key, value in params["current_state"].items(): if key not in allowed_properties: del (params["current_state"][key]) try: self.get_db() self.unmarshall_db() except Exception as exc: log.info("patch_draft() failed to get msg from db: {}".format( exc)) raise exc if not self.is_draft: raise err.PatchUnprocessable(message="this message is not a draft") try: current_state = params.pop("current_state") draft_param = Draft(params, strict=strict_patch) except Exception as exc: log.info(exc) raise err.PatchError(message=exc.message) # add missing params to be able to check consistency self_dict = self.marshall_dict() if "message_id" not in params and self.message_id: draft_param.message_id = UUIDType().to_native(self.message_id) if "parent_id" not in params and self.parent_id: draft_param.parent_id = UUIDType().to_native(self.parent_id) if "subject" not in params: draft_param.subject = self.subject if "participants" not in params and self.participants: for participant in self_dict['participants']: indexed = IndexedParticipant(participant) draft_param.participants.append(indexed) if "user_identities" not in params and self.user_identities: draft_param.user_identities = self_dict["user_identities"] # make sure the participant is present # and is consistent with selected user's identity try: new_discussion_id = draft_param.validate_consistency(user, False) except Exception as exc: log.info("consistency validation failed with err : {}".format(exc)) raise err.PatchError(message=exc.message) validated_draft = draft_param.serialize() validated_params = copy.deepcopy(params) if "participants" in params: validated_params["participants"] = validated_draft["participants"] if new_discussion_id != self.discussion_id: # discussion_id has changed, update draft's discussion_id current_state["discussion_id"] = self.discussion_id validated_params["discussion_id"] = new_discussion_id # remove empty ids from current state if any if "parent_id" in current_state and current_state["parent_id"] == "": del (current_state["parent_id"]) # handle body key mapping to body_plain or body_html # TODO: handle plain/html flag to map to right field if "body" in validated_params: validated_params["body_plain"] = validated_params["body"] del (validated_params["body"]) if "body" in current_state: current_state["body_plain"] = current_state["body"] del (current_state["body"]) # date should reflect last edit time current_state["date"] = self.date current_state["date_sort"] = self.date_sort validated_params["date"] = validated_params["date_sort"] = \ datetime.datetime.now(tz=pytz.utc) validated_params["current_state"] = current_state if "participants" in current_state and self.participants: # replace participants' label and contact_ids # because frontend has up-to-date data for these properties # which are probably not the one stored in db db_parts = {} for p in self_dict['participants']: db_parts[p['protocol'] + p['type'] + p['address']] = p for i, p in enumerate(current_state['participants']): current_state['participants'][i] = db_parts[p['protocol'] + p['type'] + p['address']] try: self.apply_patch(validated_params, **options) except Exception as exc: log.info("apply_patch() failed with error : {}".format(exc)) raise exc def unmarshall_json_dict(self, document, **options): super(Message, self).unmarshall_json_dict(document, **options) # TODO: handle html/plain flag to copy "body" key into right place if "body" in document and document["body"] is not None: self.body_plain = document["body"] def marshall_json_dict(self, **options): d = self.marshall_dict() # TODO: handle html/plain regarding user's preferences d["body"] = self.body_plain if "body_plain" in d: del (d["body_plain"]) if "body_html" in d: del (d["body_html"]) return self._json_model(d).serialize() ================================================ FILE: src/backend/main/py.main/caliopen_main/message/parameters/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .attachment import Attachment from .draft import Draft from .external_references import ExternalReferences from .message import NewMessage, NewInboundMessage, Message __all__ = ['Attachment', 'Draft', 'ExternalReferences', 'NewMessage', 'Message'] ================================================ FILE: src/backend/main/py.main/caliopen_main/message/parameters/attachment.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from schematics.models import Model from schematics.types import (StringType, UUIDType, IntType, BooleanType) class Attachment(Model): content_type = StringType() file_name = StringType() is_inline = BooleanType() size = IntType() temp_id = UUIDType() url = StringType() # objectsStore uri for temporary file (draft) or boundary reference for mime-part attachment mime_boundary = StringType() # for attachments embedded in raw messages class Options: serialize_when_none = False ================================================ FILE: src/backend/main/py.main/caliopen_main/message/parameters/draft.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals import re import uuid from schematics.types import StringType from .message import NewInboundMessage from caliopen_main.user.objects.identity import UserIdentity from caliopen_main.participant.parameters import Participant from caliopen_main.message.parameters.external_references import \ ExternalReferences from caliopen_storage.exception import NotFound from caliopen_main.common.errors import PatchUnprocessable, PatchConflict, \ PatchError from caliopen_main.message.store import Message as ModelMessage import logging log = logging.getLogger(__name__) class Draft(NewInboundMessage): body = StringType() def validate_uuid(self, user_id): if not self.message_id or not isinstance(self.message_id, uuid.UUID): raise PatchUnprocessable( message="missing or invalid message_id") try: ModelMessage.get(user_id=user_id, message_id=self.message_id) raise PatchUnprocessable(message="message_id not unique") except NotFound: pass def validate_consistency(self, user, is_new): """ Function used by create_draft and patch_draft to unsure provided params are consistent with draft's context :param user : the user object :param is_new : if true indicates that we want to validate a new draft, otherwise it is an update of existing one If needed, draft is modified to conform. """ try: self.validate() except Exception as exc: log.exception("draft validation failed with error {}".format(exc)) raise exc # copy body to body_plain TODO : manage plain or html switch user pref if hasattr(self, "body") and self.body is not None: self.body_plain = self.body else: self.body = self.body_plain = "" # fill field consistently # based on current user's selected identity from_participant = self._add_from_participant(user) if hasattr(self, 'parent_id') \ and self.parent_id is not None \ and self.parent_id != "" \ and is_new: # it is a reply, enforce participants # and other mandatory properties try: parent_msg = ModelMessage.get(user_id=user.user_id, message_id=self.parent_id) except NotFound: raise PatchError(message="parent message not found") self._build_participants_for_reply(parent_msg, from_participant) self.discussion_id = self.hash_participants self._update_external_references(user) self._build_subject_for_reply(parent_msg) elif self.discussion_id != self.hash_participants: # participants_hash has changed, update lookups self.discussion_id = self.hash_participants return self.discussion_id def _add_from_participant(self, user): if 'user_identities' not in self: raise PatchUnprocessable('Missing user identities') if len(self['user_identities']) != 1: raise PatchUnprocessable('Invalid user identities') user_identity = UserIdentity(user, identity_id=str( self['user_identities'][0])) try: user_identity.get_db() user_identity.unmarshall_db() except NotFound: raise PatchUnprocessable(message="identity not found") # add 'from' participant with local identity's identifier if not hasattr(self, 'participants'): self.participants = [] else: if len(self.participants) > 0: parts = list(self.participants) for i, participant in enumerate(parts): if re.match("from", participant['type'], re.IGNORECASE): self.participants.pop(i) from_participant = Participant() from_participant.address = user_identity.identifier.lower() from_participant.label = user_identity.display_name from_participant.protocol = user_identity.protocol from_participant.type = "From" from_participant.contact_ids = [user.contact.contact_id] self.participants.append(from_participant) # set message's protocol to sender's one if from_participant.protocol in ['email', 'smtp', 'imap']: self.protocol = 'email' else: self.protocol = from_participant.protocol return from_participant def _build_participants_for_reply(self, parent_msg, sender): """ Build participants list from message in-reply to. - former 'From' recipients are replaced by 'To' recipients - provided identity is used to fill the new 'From' participant - new sender is removed from former recipients :param sender: participant previously computed by _add_from_participant """ if not self.parent_id: return # TODO : manage reply to discussion-list # TODO : and to messages that have a `reply-to` header self.participants = [] sender['address'] = sender['address'].lower() for i, participant in enumerate(parent_msg.participants): participant['address'] = participant['address'].lower() if not re.match(sender['address'], participant['address'], re.IGNORECASE): if re.match("from", participant['type'], re.IGNORECASE): participant["type"] = "To" self.participants.append(participant) elif not re.match("list-id", participant['type'], re.IGNORECASE): self.participants.append(participant) elif not re.match("from", participant['type'], re.IGNORECASE): self.participants.append(participant) # add sender self.participants.append(sender) def _build_subject_for_reply(self, parent_msg): """ :param user: :return: """ # check subject consistency # (https://www.wikiwand.com/en/List_of_email_subject_abbreviations) # for now, we use standard prefix «Re: » (RFC5322#section-3.6.5) p = re.compile( '([\[\(] *)?(RE?S?|FYI|RIF|I|FS|VB|RV|ENC|ODP|PD|YNT|ILT|SV|VS|VL|AW|WG|ΑΠ|ΣΧΕΤ|ΠΡΘ|תגובה|הועבר|主题|转发|FWD?) *([-:;)\]][ :;\])-]*|$)|\]+ *$', re.IGNORECASE) # if hasattr(self, 'subject') and self.subject is not None: # if p.sub('', self.subject).strip() != p.sub('', # parent_msg.subject).strip(): # raise PatchConflict(message="subject has been changed") # else: # # no subject property provided : # # add subject from context with only one "Re: " prefix self.subject = "Re: " + p.sub('', parent_msg.subject, -1) def _update_external_references(self, user): """ copy externals references from current draft's ancestor and change parent_id to reflect new message's hierarchy :return: """ from caliopen_main.message.objects.message import Message parent_msg = Message(user, message_id=self.parent_id) parent_msg.get_db() parent_msg.unmarshall_db() if parent_msg: self.external_references = ExternalReferences( vars(parent_msg.external_references)) self.external_references.ancestors_ids.append( parent_msg.external_references.message_id) self.external_references.parent_id = parent_msg.external_references.message_id self.external_references.message_id = "" # will be set by broker when sending ================================================ FILE: src/backend/main/py.main/caliopen_main/message/parameters/external_references.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from schematics.models import Model from schematics.types import StringType from schematics.types.compound import ListType class ExternalReferences(Model): ancestors_ids = ListType(StringType()) message_id = StringType() parent_id = StringType() class Options: serialize_when_none = False ================================================ FILE: src/backend/main/py.main/caliopen_main/message/parameters/message.py ================================================ # -*- coding: utf-8 -*- """Caliopen parameters for message related classes.""" from __future__ import absolute_import, print_function, unicode_literals import logging from schematics.models import Model from schematics.types import (StringType, DateTimeType, IntType, UUIDType, BooleanType) from schematics.types.compound import ListType, ModelType, DictType from schematics.transforms import blacklist from .attachment import Attachment from .external_references import ExternalReferences from caliopen_main.pi.parameters import PIParameter from caliopen_main.participant.parameters import Participant from caliopen_main.user.parameters.tag import ImportedTag from caliopen_main.participant.core import hash_participants_uri import caliopen_storage.helpers.json as helpers RECIPIENT_TYPES = ['To', 'From', 'Cc', 'Bcc', 'Reply-To', 'Sender'] MESSAGE_PROTOCOLS = ['email', 'twitter', 'mastodon', None] MESSAGE_STATES = ['draft', 'sending', 'sent', 'cancel', 'unread', 'read', 'deleted'] log = logging.getLogger(__name__) class NewMessage(Model): """New message parameter.""" attachments = ListType(ModelType(Attachment), default=lambda: []) date = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') discussion_id = StringType() # = participants uris' hash external_references = ModelType(ExternalReferences) importance_level = IntType() is_answered = BooleanType() is_draft = BooleanType() is_unread = BooleanType() is_received = BooleanType() message_id = UUIDType() parent_id = UUIDType() participants = ListType(ModelType(Participant), default=lambda: []) privacy_features = DictType(StringType(), default=lambda: {}) pi = ModelType(PIParameter) raw_msg_id = UUIDType() subject = StringType() tags = ListType(StringType(), default=lambda: []) ext_tags = ListType(ModelType(ImportedTag), default=lambda: []) protocol = StringType(choices=MESSAGE_PROTOCOLS, required=False) user_identities = ListType(UUIDType(), default=lambda: []) @property def external_msg_id(self): return self.external_references.message_id if self.external_references \ else None class Options: serialize_when_none = False @property def hash_participants(self): ids_hash = hash_participants_uri(self.participants) return ids_hash['hash'] class NewInboundMessage(NewMessage): body_html = StringType() body_plain = StringType() class Message(NewInboundMessage): """Existing message parameter.""" body = StringType() user_id = UUIDType() date_insert = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') date_delete = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') date_sort = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') class Options: roles = {'default': blacklist('user_id', 'date_delete')} serialize_when_none = False ================================================ FILE: src/backend/main/py.main/caliopen_main/message/parsers/__init__.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/message/parsers/mail.py ================================================ # -*- coding: utf-8 -*- """ Caliopen mail message format management. mail parsing is included in python, so this is not getting external dependencies. For formats with needs of external packages, they must be defined outside of this one. """ import logging import base64 import string from itertools import groupby from mailbox import Message from email.header import decode_header import datetime import pytz from email.utils import parsedate_tz, mktime_tz, getaddresses import zope.interface from caliopen_main.common.helpers.normalize import clean_email_address from caliopen_main.common.helpers.strings import to_utf8 from caliopen_main.common.interfaces import (IAttachmentParser, IMessageParser, IParticipantParser) # from caliopen_main.common.objects.tag import ResourceTag as Tag from caliopen_main.user.parameters.tag import ImportedTag as Tag log = logging.getLogger(__name__) TEXT_CONTENT_TYPE = ['text', 'xml', 'vnd', 'xhtml', 'json', 'msword'] EXCLUDED_EXT_FLAGS = ['\Seen', 'nonjunk', '$notjunk', 'notjunk', '$mdnsent', '$forwarded', '$sent', '\Recent', '\All', '\Archive', '\Drafts', '\Junk', '\Sent', '\Trash'] class MailAttachment(object): """Mail part structure.""" zope.interface.implements(IAttachmentParser) def __init__(self, part): """Extract attachment attributes from a mail part.""" self.content_type = part.get_content_type() self.filename = part.get_filename() if self.filename: try: self.filename = self.filename.decode('utf-8') except UnicodeError: log.warn('Invalid filename encoding') content_disposition = part.get("Content-Disposition") if content_disposition: dispositions = content_disposition.strip().split(";") self.is_inline = bool(dispositions[0].lower() == "inline") else: self.is_inline = True data = part.get_payload() self.can_index = False if any(x in part.get_content_type() for x in TEXT_CONTENT_TYPE): self.can_index = True charsets = part.get_charsets() if len(charsets) > 1: raise Exception('Too many charset %r for %s' % (charsets, part.get_payload())) self.charset = charsets[0] if 'Content-Transfer-Encoding' in part.keys(): if part.get('Content-Transfer-Encoding') == 'base64': data = base64.b64decode(data) if self.charset: data = data.decode(self.charset, 'replace'). \ encode('utf-8') boundary = part.get("Mime-Boundary", failobj="") if boundary is not "": self.mime_boundary = boundary else: self.mime_boundary = "" self.data = data self.size = len(data) if data else 0 @classmethod def is_attachment(cls, part): """Check if a part conform to Caliopen's attachment definition. A part is an "attachment" if it verifies ANY of this conditions : - it has a Content-Disposition header with param "attachment" - the main part of the Content-Type header is within "attachment_types" list below - the part is not a PGP/Mime encryption envelope see https://www.iana.org/assignments/media-types/media-types.xhtml :param part: an email/message's part as return by the walk() func. :return: true or false """ content_disposition = part.get("Content-Disposition") if content_disposition: dispositions = content_disposition.strip().split(";") if bool(dispositions[0].lower() == "attachment") or \ bool(dispositions[0].lower() == "inline"): return True attachment_types = ( "application", "image", "video", "audio", "message", "font") if part.get_content_maintype() in attachment_types: return True return False class MailParticipant(object): """Mail participant parser.""" zope.interface.implements(IParticipantParser) def __init__(self, type, addr): """Parse an email address and create a participant.""" self.type = type parts = clean_email_address(addr) self.address = parts[0] self.label = parts[1] class MailMessage(object): """ Mail message structure. Got a mail in raw rfc2822 format, parse it to resolve all recipients emails, parts and group headers """ zope.interface.implements(IMessageParser) recipient_headers = ['From', 'To', 'Cc', 'Bcc'] message_protocol = 'email' warnings = [] body_html = "" body_plain = "" def __init__(self, raw_data): """Parse an RFC2822,5322 mail message.""" self.raw = raw_data self._extra_parameters = {} try: self.mail = Message(raw_data) except Exception as exc: log.error('Parse message failed %s' % exc) raise exc if self.mail.defects: # XXX what to do ? log.warn('Defects on parsed mail %r' % self.mail.defects) self.warning = self.mail.defects self.get_bodies() def get_bodies(self): """Extract body alternatives, if any.""" body_html = "" body_plain = "" if self.mail.get("Content-Type", None): if self.mail.is_multipart(): if self.mail.get_content_subtype() == 'encrypted': parts = self.mail.get_payload() if len(parts) == 2: self.body_plain = parts[1].get_payload() return else: log.warn('Encrypted message with invalid parts count') for top_level_part in self.mail.get_payload(): if top_level_part.get_content_maintype() == "multipart": for alternative in top_level_part.get_payload(): charset = alternative.get_param("charset") if isinstance(charset, tuple): charset = unicode(charset[2], charset[0] or "us-ascii") if alternative.get_content_type() == "text/plain": body_plain = alternative.get_payload( decode=True) self.body_plain = to_utf8(body_plain, charset) elif alternative.get_content_type() == "text/html": body_html = alternative. \ get_payload(decode=True) self.body_html = to_utf8(body_html, charset) break else: charset = top_level_part.get_param("charset") if isinstance(charset, tuple): charset = unicode(charset[2], charset[0] or "us-ascii") if top_level_part.get_content_type() == "text/plain": body_plain = top_level_part. \ get_payload(decode=True) self.body_plain = to_utf8(body_plain, charset) elif top_level_part.get_content_type() == "text/html": body_html = top_level_part.get_payload(decode=True) self.body_html = to_utf8(body_html, charset) else: charset = self.mail.get_param("charset") if isinstance(charset, tuple): charset = unicode(charset[2], charset[0] or "us-ascii") if self.mail.get_content_type() == "text/html": body_html = self.mail.get_payload(decode=True) self.body_html = to_utf8(body_html, charset) else: body_plain = self.mail.get_payload(decode=True) self.body_plain = to_utf8(body_plain, charset) else: self.body_plain = self.mail.get_payload(decode=True) @property def subject(self): """Mail subject.""" s = decode_header(self.mail.get('Subject')) charset = s[0][1] if charset is not None: return s[0][0].decode(charset, "replace"). \ encode("utf-8", "replace") else: try: return s[0][0].decode('utf-8', errors='ignore') except UnicodeError: log.warn('Invalid subject encoding') return s[0][0] @property def size(self): """Get mail size in bytes.""" return len(self.mail.as_string()) @property def external_references(self): """Return mail references to be used as external references. making use of RFC5322 headers : message-id in-reply-to references headers' strings are pruned to extract email addresses only. """ ext_id = self.mail.get('Message-Id') parent_id = self.mail.get('In-Reply-To') ref = self.mail.get_all("References") ref_addr = getaddresses(ref) if ref else None ref_ids = [address[1] for address in ref_addr] if ref_addr else [] mid = clean_email_address(ext_id)[1] if ext_id else None if not mid: log.error('Unable to find correct message_id {}'.format(ext_id)) mid = ext_id pid = clean_email_address(parent_id)[1] if parent_id else None if not pid: pid = parent_id return { 'message_id': mid, 'parent_id': pid, 'ancestors_ids': ref_ids} @property def date(self): """Get UTC date from a mail message.""" mail_date = self.mail.get('Date') if mail_date: try: tmp_date = parsedate_tz(mail_date) return datetime.datetime.fromtimestamp(mktime_tz(tmp_date)) except TypeError: log.error('Invalid date in mail {}'.format(mail_date)) log.debug('No date on mail using now (UTC)') return datetime.datetime.now(tz=pytz.utc) @property def participants(self): """Mail participants.""" participants = [] for header in self.recipient_headers: addrs = [] participant_type = header.capitalize() if self.mail.get(header): parts = self.mail.get(header).split('>,') if not parts: pass if parts and parts[0] == 'undisclosed-recipients:;': pass filtered = [x for x in parts if '@' in x] addrs.extend(filtered) for addr in addrs: participant = MailParticipant(participant_type, addr.lower()) if participant.address == '' and participant.label == '': log.warn('Invalid email address {}'.format(addr)) else: participants.append(participant) return participants @property def attachments(self): """Extract parts which we consider as attachments.""" if not self.mail.is_multipart(): return [] attchs = [] for p in walk_with_boundary(self.mail, ""): if not p.is_multipart(): if p.get_content_subtype() == 'pgp-encrypted': # Special consideration. Do not present it as an attachment # but set _extra_parameters accordingly self._extra_parameters.update({'encrypted': 'pgp'}) continue if MailAttachment.is_attachment(p): attchs.append(MailAttachment(p)) return attchs @property def extra_parameters(self): """Mail message extra parameters.""" lists = self.mail.get_all("List-ID") lists_addr = getaddresses(lists) if lists else None lists_ids = [address[1] for address in lists_addr] \ if lists_addr else [] self._extra_parameters.update({'lists': lists_ids}) return self._extra_parameters # Others parameters specific for mail message @property def headers(self): """Extract all headers into list. Duplicate on headers exists, group them by name with a related list of values """ def keyfunc(item): return item[0] # Group multiple value for same headers into a dict of list headers = {} data = sorted(self.mail.items(), key=keyfunc) for k, g in groupby(data, key=keyfunc): headers[k] = [x[1] for x in g] return headers @property def external_flags(self): """ Get headers added by our fetcher that represent flags or labels set by external provider, returned as list of tags """ tags = [] for h in ['X-Fetched-Imap-Flags', 'X-Fetched-X-GM-LABELS']: enc_flags = self.mail.get(h) if enc_flags: flags_str = base64.decodestring(enc_flags) for flag in string.split(flags_str, '\r\n'): if flag not in EXCLUDED_EXT_FLAGS: tag = Tag() tag.name = flag tag.label = flag tag.type = 'imported' tags.append(tag) return tags def walk_with_boundary(message, boundary): """Recurse in boundaries.""" message.add_header("Mime-Boundary", boundary) yield message if message.is_multipart(): subboundary = message.get_boundary("") for subpart in message.get_payload(): for subsubpart in walk_with_boundary(subpart, subboundary): yield subsubpart ================================================ FILE: src/backend/main/py.main/caliopen_main/message/parsers/mastodon.py ================================================ # -*- coding: utf-8 -*- import logging import json import dateutil.parser import zope.interface from caliopen_main.common.interfaces import (IMessageParser, IParticipantParser) from caliopen_main.common.helpers.normalize import parse_mastodon_url log = logging.getLogger(__name__) class MastodonStatus(object): """ Mastodon status structure """ zope.interface.implements(IMessageParser) message_protocol = 'mastodon' warnings = [] body_html = "" body_plain = "" def __init__(self, raw_data): self.raw = raw_data self.dm = json.loads(self.raw) self.recipients = [] for m in self.dm["mentions"]: self.recipients.append(MastodonParticipant("To", m['url'])) self.sender = MastodonParticipant('From', self.dm['account']['url']) self.protocol = self.message_protocol self.is_unread = True # TODO: handle DM sent by user # if broker keeps them when fetching self.is_draft = False self.is_answered = False self.is_received = True # TODO: handle DM sent by user # if broker keeps them when fetching self.importance_level = 0 self.get_bodies() def get_bodies(self): if self.dm['spoiler_text'] != '': self.body_html = "" + self.dm[ 'spoiler_text'] + "" self.body_html += self.dm['content'] @property def subject(self): """ toots don't have subject should we return an excerpt ? """ return '' @property def size(self): """Get json toot object size in bytes.""" return len(self.dm.as_string()) @property def date(self): return dateutil.parser.isoparse(self.dm['created_at'].rstrip('UTC')) @property def participants(self): "one sender only for now" p = [self.sender] p.extend(self.recipients) return p @property def external_references(self): return {'message_id': self.dm["id"], 'parent_id': self.dm["in_reply_to_id"]} @property def attachments(self): """TODO""" return [] @property def extra_parameters(self): """TODO""" return {} class MastodonParticipant(object): """ Mastodon sender and recipient parser """ zope.interface.implements(IParticipantParser) def __init__(self, type, url): """Parse a mastodon address and create a participant.""" domain, username = parse_mastodon_url(url) self.address = username + '@' + domain self.label = username self.type = type ================================================ FILE: src/backend/main/py.main/caliopen_main/message/parsers/twitter.py ================================================ # -*- coding: utf-8 -*- import logging import json from datetime import datetime import pytz import zope.interface from caliopen_main.common.interfaces import (IMessageParser, IParticipantParser) from caliopen_main.common.helpers.normalize import clean_twitter_address log = logging.getLogger(__name__) class TwitterDM(object): """ Twitter direct message structure """ zope.interface.implements(IMessageParser) recipient_headers = ['From', 'To'] message_protocol = 'twitter' warnings = [] body_html = "" body_plain = "" def __init__(self, raw_data): self.raw = raw_data self.dm = json.loads(self.raw) self.recipient_name = self.dm["message_create"]["target"][ "recipient_screen_name"] self.sender_name = self.dm["message_create"]["sender_screen_name"] self.protocol = self.message_protocol self.is_unread = True # TODO: handle DM sent by user # if broker keeps them when fetching self.is_draft = False self.is_answered = False self.is_received = True # TODO: handle DM sent by user # if broker keeps them when fetching self.importance_level = 0 self.get_bodies() def get_bodies(self): self.body_plain = self.dm["message_create"]["message_data"]["text"] @property def subject(self): """ tweets don't have subject should we return an excerpt ? """ return '' @property def size(self): """Get json tweet object size in bytes.""" return len(self.dm.as_string()) @property def date(self): return datetime.fromtimestamp(float(self.dm["created_timestamp"])/1000, tz=pytz.utc) @property def participants(self): "one sender only for now" return [TwitterParticipant("To", clean_twitter_address(self.recipient_name)), TwitterParticipant("From", clean_twitter_address(self.sender_name))] @property def external_references(self): return {'message_id': self.dm["id"]} @property def attachments(self): """TODO""" return [] @property def extra_parameters(self): """TODO""" return {} class TwitterParticipant(object): """ Twitter sender and recipient parser """ zope.interface.implements(IParticipantParser) def __init__(self, type, screen_name): """Parse a twitter address and create a participant.""" self.type = type self.address = clean_twitter_address(screen_name) self.label = screen_name ================================================ FILE: src/backend/main/py.main/caliopen_main/message/store/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .attachment import MessageAttachment from .attachment_index import IndexedMessageAttachment from .external_references import ExternalReferences, MessageExternalRefLookup from .external_references_index import IndexedExternalReferences from .message import Message from .message_index import IndexedMessage from caliopen_main.participant.store.participant import Participant from .raw import RawMessage, UserRawLookup __all__ = ['MessageAttachment', 'IndexedMessageAttachment', 'RawMessage', 'UserRawLookup', 'Message', 'IndexedMessage', 'ExternalReferences', 'IndexedExternalReferences', 'Participant', 'IndexedParticipant', 'MessageExternalRefLookup' ] ================================================ FILE: src/backend/main/py.main/caliopen_main/message/store/attachment.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from cassandra.cqlengine import columns from caliopen_storage.store import BaseUserType class MessageAttachment(BaseUserType): """Attachment nested in message.""" content_type = columns.Text() file_name = columns.Text() is_inline = columns.Boolean() size = columns.Integer() temp_id = columns.UUID() url = columns.Text() # objectsStore uri for temporary file (draft) mime_boundary = columns.Text() # for attachments embedded in raw messages ================================================ FILE: src/backend/main/py.main/caliopen_main/message/store/attachment_index.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals import logging from elasticsearch_dsl import InnerObjectWrapper, Boolean, Integer, Keyword log = logging.getLogger(__name__) class IndexedMessageAttachment(InnerObjectWrapper): """Nest attachment indexed model.""" content_type = Keyword() file_name = Keyword() is_inline = Boolean() size = Integer() temp_id = Keyword() url = Keyword() # objectsStore uri for temporary file (draft) mime_boundary = Keyword() # for attachments embedded in raw messages ================================================ FILE: src/backend/main/py.main/caliopen_main/message/store/external_references.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from cassandra.cqlengine import columns from caliopen_storage.store import BaseUserType from caliopen_storage.store.model import BaseModel class ExternalReferences(BaseUserType): """External references nested in message.""" ancestors_ids = columns.List(columns.Text()) message_id = columns.Text() parent_id = columns.Text() class MessageExternalRefLookup(BaseModel): """Table to lookup message by external message-id""" user_id = columns.UUID(primary_key=True) external_msg_id = columns.Text(primary_key=True) identity_id = columns.UUID(primary_key=True) message_id = columns.UUID() ================================================ FILE: src/backend/main/py.main/caliopen_main/message/store/external_references_index.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals import logging from elasticsearch_dsl import InnerObjectWrapper, Keyword, Nested log = logging.getLogger(__name__) class IndexedExternalReferences(InnerObjectWrapper): """Nest attachment indexed model.""" ancestors_ids = Keyword(multi=True) message_id = Keyword() parent_id = Keyword() ================================================ FILE: src/backend/main/py.main/caliopen_main/message/store/message.py ================================================ # -*- coding: utf-8 -*- """Caliopen storage model for messages.""" from __future__ import absolute_import, print_function, unicode_literals from cassandra.cqlengine import columns from caliopen_storage.store.model import BaseModel from caliopen_storage.store.mixin import IndexedModelMixin from caliopen_main.pi.objects import PIModel from .attachment import MessageAttachment from .external_references import ExternalReferences from caliopen_main.participant.store.participant import Participant from .message_index import IndexedMessage import uuid class Message(BaseModel, IndexedModelMixin): """Message model.""" _index_class = IndexedMessage user_id = columns.UUID(primary_key=True) message_id = columns.UUID(primary_key=True, default=uuid.uuid4) attachments = columns.List(columns.UserDefinedType(MessageAttachment)) body_html = columns.Text() body_plain = columns.Text() date = columns.DateTime() date_delete = columns.DateTime() date_insert = columns.DateTime() date_sort = columns.DateTime() discussion_id = columns.Text() external_references = columns.UserDefinedType(ExternalReferences) importance_level = columns.Integer() is_answered = columns.Boolean() is_draft = columns.Boolean() is_unread = columns.Boolean() is_received = columns.Boolean(default=False) parent_id = columns.UUID() participants = columns.List(columns.UserDefinedType(Participant)) privacy_features = columns.Map(columns.Text(), columns.Text()) pi = columns.UserDefinedType(PIModel) raw_msg_id = columns.UUID() subject = columns.Text() # Subject of email, the message for short tags = columns.List(columns.Text(), db_field="tagnames") protocol = columns.Text() user_identities = columns.List(columns.UUID) ================================================ FILE: src/backend/main/py.main/caliopen_main/message/store/message_index.py ================================================ # -*- coding: utf-8 -*- """Caliopen message index classes.""" from __future__ import absolute_import, print_function, unicode_literals from elasticsearch_dsl import Mapping, Nested, Text, Keyword, Date, Boolean, \ Integer, Object from caliopen_storage.store.model import BaseIndexDocument from .attachment_index import IndexedMessageAttachment from .external_references_index import IndexedExternalReferences from caliopen_main.pi.objects import PIIndexModel from caliopen_main.participant.store.participant_index import IndexedParticipant class IndexedMessage(BaseIndexDocument): """Contact indexed message model.""" doc_type = 'indexed_message' user_id = Keyword() message_id = Keyword() attachments = Nested(doc_class=IndexedMessageAttachment) body_html = Text() body_plain = Text() date = Date() date_delete = Date() date_insert = Date() date_sort = Date() discussion_id = Keyword() external_references = Nested(doc_class=IndexedExternalReferences) importance_level = Integer() is_answered = Boolean() is_draft = Boolean() is_unread = Boolean() is_received = Boolean() parent_id = Keyword() participants = Nested(doc_class=IndexedParticipant) privacy_features = Object() pi = Object(doc_class=PIIndexModel) raw_msg_id = Keyword() subject = Text() tags = Keyword(multi=True) protocol = Keyword() user_identities = Keyword(multi=True) @property def message_id(self): """The compound primary key for a message is message_id.""" return self.meta.id @classmethod def build_mapping(cls): """Generate the mapping definition for indexed messages""" m = Mapping(cls.doc_type) m.meta('_all', enabled=True) m.field('user_id', 'keyword') # attachments m.field('attachments', Nested(doc_class=IndexedMessageAttachment, include_in_all=True, properties={ "content_type": Keyword(), "file_name": Keyword(), "is_inline": Boolean(), "size": Integer(), "temp_id": Keyword(), "url": Keyword(), "mime_boundary": Keyword() }) ) m.field('body_html', 'text', fields={ "normalized": {"type": "text", "analyzer": "text_analyzer"} }) m.field('body_plain', 'text', fields={ "normalized": {"type": "text", "analyzer": "text_analyzer"} }) m.field('date', 'date') m.field('date_delete', 'date') m.field('date_insert', 'date') m.field('date_sort', 'date') m.field('discussion_id', 'keyword') # external references m.field('external_references', Nested(doc_class=IndexedExternalReferences, include_in_all=True, properties={ "ancestors_ids": Keyword(), "message_id": Keyword(), "parent_id": Keyword() }) ) m.field('importance_level', 'short') m.field('is_answered', 'boolean') m.field('is_draft', 'boolean') m.field('is_unread', 'boolean') m.field('is_received', 'boolean') m.field('message_id', 'keyword') m.field('parent_id', 'keyword') # participants participants = Nested(doc_class=IndexedParticipant, include_in_all=True) participants.field("address", "text", analyzer="text_analyzer", fields={ "raw": {"type": "keyword"}, "parts": {"type": "text", "analyzer": "email_analyzer"} }) participants.field("contact_ids", Keyword(multi=True)) participants.field("label", "text", analyzer="text_analyzer") participants.field("protocol", Keyword()) participants.field("type", Keyword()) m.field('participants', participants) # PI pi = Object(doc_class=PIIndexModel, include_in_all=True, properties={ "technic": "integer", "comportment": "integer", "context": "integer", "version": "integer", "date_update": "date" }) m.field("pi", pi) m.field('privacy_features', Object(include_in_all=True)) m.field('raw_msg_id', "keyword") m.field('subject', 'text', fields={ "normalized": {"type": "text", "analyzer": "text_analyzer"} }) m.field('tags', Keyword(multi=True)) m.field('subject', 'text') m.field('tags', Keyword(multi=True)) m.field('protocol', 'keyword') m.field('user_identities', Keyword(multi=True)) return m ================================================ FILE: src/backend/main/py.main/caliopen_main/message/store/raw.py ================================================ # -*- coding: utf-8 -*- """Caliopen storage model for messages.""" from __future__ import absolute_import, print_function, unicode_literals import uuid from cassandra.cqlengine import columns from caliopen_storage.store.model import BaseModel class RawMessage(BaseModel): """Raw message model.""" raw_msg_id = columns.UUID(primary_key=True, default=uuid.uuid4) raw_data = columns.Bytes() # may be empty if data is too large to fit into cassandra raw_size = columns.Integer() # number of bytes in 'data' column uri = columns.Text() # where object is stored if it was too large to fit into raw_data column delivered = columns.Boolean() # true only if complete delivery succeeded class UserRawLookup(BaseModel): """User's raw message pointer.""" user_id = columns.UUID(primary_key=True) raw_msg_id = columns.UUID(primary_key=True) ================================================ FILE: src/backend/main/py.main/caliopen_main/notification/__init__.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/notification/core.py ================================================ # -*- coding: utf-8 -*- """Caliopen device core classes.""" from __future__ import absolute_import, print_function, unicode_literals from caliopen_storage.core import BaseCore from caliopen_main.common.core import BaseUserCore from .store import Notification as ModelNotification, \ NotificationTtl as ModelTTLs class Notification(BaseUserCore): """User Notification core class""" _model_class = ModelNotification _pkey_name = "user_id" class NotificationTtl(BaseCore): """core class to store default TTLs for each notification kind""" _model_class = ModelTTLs _pkey_name = "notif_code" ================================================ FILE: src/backend/main/py.main/caliopen_main/notification/store.py ================================================ # -*- coding: utf-8 -*- """Caliopen cassandra models related to device.""" from __future__ import absolute_import, print_function, unicode_literals from cassandra.cqlengine import columns from caliopen_storage.store.model import BaseModel class Notification(BaseModel): """ Table to store notifications queues in cassandra user_id: user's id to which notification belongs to. notif_id: time UUID V1 (en.wikipedia.org/wiki/Universally_unique_identifier) emitter : backend entity that's emitting the message. type: a single word to describe notification's type: event, info, feedback.. reference: (optional) a reference number previously sent by frontend. body: could be a simple word or a more complex structure like a json. """ user_id = columns.UUID(primary_key=True) notif_id = columns.TimeUUID(primary_key=True) emitter = columns.Text() type = columns.Ascii() reference = columns.Text() body = columns.Blob() class NotificationTtl(BaseModel): """ Table to store ttl configuration for each kind of notification ttl_code: chars to identify a ttl. ttl_duration: default duration for this kind of ttl description: free text description """ ttl_code = columns.Ascii(primary_key=True) ttl_duration = columns.Integer() description = columns.Text() ================================================ FILE: src/backend/main/py.main/caliopen_main/participant/__init__.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/participant/core/__init__.py ================================================ from .participant import hash_participants_uri, participants_from_uris, \ HashLookup, ParticipantHash __all__ = ['hash_participants_uri', 'participants_from_uris', 'HashLookup', 'ParticipantHash'] ================================================ FILE: src/backend/main/py.main/caliopen_main/participant/core/participant.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals import hashlib import logging from datetime import datetime from caliopen_storage.exception import NotFound from caliopen_storage.core import BaseCore from caliopen_main.participant.store import HashLookup as ModelHashLookup, \ ParticipantHash as ModelParticipantHash log = logging.getLogger(__name__) class HashLookup(BaseCore): _model_class = ModelHashLookup _pkey_name = 'hash' class ParticipantHash(BaseCore): _model_class = ModelParticipantHash _pkey_name = 'value' def hash_participants_uri(participants): """ Create hash from a collection of Participant :param participants: a collection of Participant :return: sorted collection of participants' URI + hash of this collection hash is computed from a set of URIs which are strings modeled as 'participant.protocol:participant.address' """ URIs = set() for participant in participants: if not participant.address or not participant.protocol: raise Exception("missing mandatory property in participant") uri = participant.protocol + ":" + participant.address.lower() URIs.add(uri) URIs = list(URIs) URIs.sort() hash = hashlib.sha256(''.join(URIs)).hexdigest() return {'uris': URIs, 'hash': hash} def participants_from_uris(user, uris, uris_hash): """ - resolve uris to contact to build participants' set - compute participants_hash - create two ways links : uris<->uris_hash uris<->participants_hash :param user: :param uris: a set() of uris formatted like 'scheme:path' :param uris_hash: the hash or uris' components :type uris: set :type uris_hash: string :return: participant """ from caliopen_main.contact.core import ContactLookup participants = set() for uri in uris: try: contact = ContactLookup.get(user, uri.split(":", 1)[1]) if contact: participants.add("contact:" + contact.contact_id) else: participants.add(uri) except NotFound: participants.add(uri) participants = list(participants) participants.sort() participants_hash = hashlib.sha256(''.join(participants)).hexdigest() date = datetime.utcnow() # store uris_hash -> participants_hash ParticipantHash.create(user_id=user.user_id, kind="uris", key=uris_hash, value=participants_hash, components=uris, date_insert=date) # store participants_hash -> uris_hash ParticipantHash.create(user_id=user.user_id, kind="participants", key=participants_hash, value=uris_hash, components=participants, date_insert=date) for uri in uris: # uri->uris_hash HashLookup.create(user_id=user.user_id, uri=uri, hash=uris_hash, hash_components=uris, date_insert=date) return {'components': participants, 'hash': participants_hash} ================================================ FILE: src/backend/main/py.main/caliopen_main/participant/objects/__init__.py ================================================ from .participant import Participant __all__ = ['Participant'] ================================================ FILE: src/backend/main/py.main/caliopen_main/participant/objects/participant.py ================================================ # -*- coding: utf-8 -*- """Caliopen message object classes.""" from __future__ import absolute_import, print_function, unicode_literals import types from uuid import UUID from caliopen_main.common.objects.base import ObjectJsonDictifiable from caliopen_main.participant.store.participant import \ Participant as ModelParticipant from caliopen_main.participant.store.participant_index import IndexedParticipant class Participant(ObjectJsonDictifiable): """participant's attributes, nested within message object""" _attrs = { 'address': types.StringType, 'contact_ids': [UUID], 'label': types.StringType, 'participant_id': UUID, 'protocol': types.StringType, 'type': types.StringType } _model_class = ModelParticipant _index_class = IndexedParticipant ================================================ FILE: src/backend/main/py.main/caliopen_main/participant/parameters/__init__.py ================================================ from .participant import Participant __all__ = ['Participant'] ================================================ FILE: src/backend/main/py.main/caliopen_main/participant/parameters/participant.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals import logging from schematics.models import Model from schematics.types import StringType, UUIDType from schematics.types.compound import ListType log = logging.getLogger(__name__) class Participant(Model): address = StringType() contact_ids = ListType(UUIDType(), default=lambda: []) label = StringType() protocol = StringType() type = StringType() participant_id = UUIDType() class Options: serialize_when_none = False ================================================ FILE: src/backend/main/py.main/caliopen_main/participant/store/__init__.py ================================================ from .participant import Participant, ParticipantHash, HashLookup from .participant_index import IndexedParticipant __all__ = ['Participant', 'ParticipantHash', 'IndexedParticipant', 'HashLookup'] ================================================ FILE: src/backend/main/py.main/caliopen_main/participant/store/participant.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from cassandra.cqlengine import columns from caliopen_storage.store import BaseUserType, BaseModel class Participant(BaseUserType): """participant nested in message.""" address = columns.Text() contact_ids = columns.List(columns.UUID()) participant_id = columns.UUID() label = columns.Text() protocol = columns.Text() type = columns.Text() class HashLookup(BaseModel): """ Table to lookup in which hash(es) an uri is embedded URIs in the form of "scheme:path" - for example : "email:john@example.com", "twitter:caliopen_org" It is updated each time a message gets in or out (including draft edition) """ user_id = columns.UUID(primary_key=True) uri = columns.Text(primary_key=True) hash = columns.Text(primary_key=True) hash_components = columns.List(columns.Text()) date_insert = columns.DateTime() class ParticipantHash(BaseModel): """ Table to store two ways links between uris'hash (immutable message's prop.) and corresponding current participants'hash It is updated each time : - a lookup is made on one uris'hash, but its participant hash counterpart does not exist - a participant is added/removed from a contact """ user_id = columns.UUID(primary_key=True) kind = columns.Text(primary_key=True) # 'uris' or 'participants' key = columns.Text(primary_key=True) # uris or partcipants' hash value = columns.Text(primary_key=True) # the hash of opposite kind components = columns.List(columns.Text()) # what hash is made of date_insert = columns.DateTime() ================================================ FILE: src/backend/main/py.main/caliopen_main/participant/store/participant_index.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals import logging from elasticsearch_dsl import InnerObjectWrapper, Keyword, Text log = logging.getLogger(__name__) class IndexedParticipant(InnerObjectWrapper): """Nest participant indexed model.""" address = Keyword() contact_ids = Keyword(multi=True) label = Text() participant_id = Keyword() protocol = Keyword() type = Keyword() ================================================ FILE: src/backend/main/py.main/caliopen_main/pi/__init__.py ================================================ from .parameters import PIParameter from .objects import PIModel, PIIndexModel, PIObject __all__ = ['PIParameter', 'PIModel', 'PIIndexModel', 'PIObject'] ================================================ FILE: src/backend/main/py.main/caliopen_main/pi/objects.py ================================================ # -*- coding: utf-8 -*- """ Caliopen PI (privacy indexes) definition. This structure is common to many entities (user, contact, message) """ from __future__ import absolute_import, print_function, unicode_literals import types from uuid import UUID from cassandra.cqlengine import columns from elasticsearch_dsl import InnerObjectWrapper, Integer, Date from caliopen_storage.store import BaseUserType from caliopen_main.common.objects.base import ObjectIndexable class PIModel(BaseUserType): """The privacy indexes model definition.""" technic = columns.Integer(default=0) comportment = columns.Integer(default=0) context = columns.Integer(default=0) version = columns.Integer(default=0) date_update = columns.DateTime() class PIIndexModel(InnerObjectWrapper): """The privacy indexes model definition for index part.""" comportment = Integer() context = Integer() date_update = Date() technic = Integer() version = Integer() class PIObject(ObjectIndexable): """The caliopen object definition of privacy indexes.""" _attrs = { "technic": types.IntType, "comportment": types.IntType, "context": types.IntType, "version": types.IntType, "user_id": UUID } _model_class = PIModel _index_class = PIIndexModel ================================================ FILE: src/backend/main/py.main/caliopen_main/pi/parameters.py ================================================ # -*- coding: utf-8 -*- """ Caliopen PI (privacy indexes) definition. This structure is common to many entities (user, contact, message) """ from __future__ import absolute_import, print_function, unicode_literals from schematics.models import Model from schematics.types import IntType, DateTimeType class PIParameter(Model): """The privacy indexes schematics parameter definition.""" technic = IntType() comportment = IntType() context = IntType() version = IntType() date_update = DateTimeType() ================================================ FILE: src/backend/main/py.main/caliopen_main/protocol/__init__.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/protocol/core/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .provider import Provider __all__ = [ 'Provider' ] ================================================ FILE: src/backend/main/py.main/caliopen_main/protocol/core/provider.py ================================================ # -*- coding: utf-8 -*- """Caliopen device core classes.""" from __future__ import absolute_import, print_function, unicode_literals from caliopen_storage.core import BaseCore from ..store.provider import Provider as ModelProvider class Provider(BaseCore): """Provider core class""" _model_class = ModelProvider _pkey_name = "name" ================================================ FILE: src/backend/main/py.main/caliopen_main/protocol/store/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .provider import Provider __all__ = [ 'Provider' ] ================================================ FILE: src/backend/main/py.main/caliopen_main/protocol/store/provider.py ================================================ # -*- coding: utf-8 -*- """Caliopen tag objects.""" from __future__ import absolute_import, print_function, unicode_literals from cassandra.cqlengine import columns from caliopen_storage.store.model import BaseModel class Provider(BaseModel): """model to store data related to external protocol endpoints""" name = columns.Text(primary_key=True) instance = columns.Text(primary_key=True) infos = columns.Map(columns.Text, columns.Text) date_insert = columns.DateTime() ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/__init__.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/fixtures/mail/pgp_crypted_1.eml ================================================ Message-ID: <1492761665.964.1.camel@gandi.net> Subject: crypted content From: Chamal To: Caliopen Date: Fri, 21 Apr 2017 10:01:05 +0200 Content-Type: multipart/encrypted; protocol="application/pgp-encrypted"; boundary="=-hf6zgIn8mtv3IPX/X7Tq" X-Mailer: Evolution 3.22.6 Mime-Version: 1.0 X-Evolution-Identity: 1485870052.1665.0@tao X-Evolution-Fcc: folder://1485870052.1665.2%40tao/Sent X-Evolution-Transport: 1485870052.1665.10@tao X-Evolution-Source: --=-hf6zgIn8mtv3IPX/X7Tq Content-Type: application/pgp-encrypted Content-Transfer-Encoding: 7bit Version: 1 --=-hf6zgIn8mtv3IPX/X7Tq Content-Type: application/octet-stream; name="encrypted.asc" Content-Description: Ceci est une partie de message =?ISO-8859-1?Q?num=E9riquement?= =?ISO-8859-1?Q?_sign=E9e?= Content-Transfer-Encoding: 7bit -----BEGIN PGP MESSAGE----- hQIMAxEK1/MMDKzXAQ/+KpNeJBR9o5etazFr16HO/vWAdDlkq3iIu7BOm0sAXYIJ ssjXN9cXY1LM1B70GeXvGeB4suqJBjrl6xbKCW0fMQsGovYQpyHJAOJCtModM6vM 76OuuTcT0AdE40pgfscTdG3nCUV2T/w2IKg7TNLwEstRQ4BoA+RFPyar97Zq1Pyk 74Wkwv1BT9jMejkpaQycqlV1eTF/JNbXa/hs5dcmirbH9PCoySQx+qFnp4w4ZHnx JDT/MpmK4M3w7FawyA2adUFKeBqXU9Oo2LBz9f9ZBtIPHeiV2iklEgQUqv/847dc OqB23fd+Jq8mXjmycLcwEsXjOL77dI+SCip4MheFvGjwSOUoIkIb6SbcnUq/hwsi xpLc3yeZDm4BCi/M0crmhMF8NEJYyu0Xg2Si+lWtWPHCtrXW+Lla226BcGNlV1Dx cpaeMlFF2MvADqTsxAD8F+km2mrf7RznobnX70TkH2wF80W+/KIKbc0R2ilyvyIm Y/MgttT593nUJgIm77cIzUjSMJHVfhYASS960Rv4cvKMIYcJQ41zJ1O693eB8gXM ebCU7Tg6gzZ1Fckgil0VQyRwY9cPERZQSrgcQOdZaw51jYzJNrRDXaaX8TuMnSTx xJkalCh8PEJ9mtLIvNPgw5cGNhttNkoGvPamEAXEHTNE7757qo5ku7w4X2u9t6uF AQwD08kNfkWMEY4BB/9N42dz6IwZ7GG/9Hj0Avcb5l8If1peNj/zOUXv88JBvMEL GukefhD2ez0R359ErrJk4Q+RhOOJYkrcPUShqm0SN7s50TVsms5/7T/OT0mxVsOR 6nnm19DMGW32Kz819c8Gp+kVB7sFOldJpi/kyoXJsO/0HdnXaM4HyeF8csTFFvLf B2bZFNk5YgasiCbCys2M0Zv8MtmUXQc7y1xsHSvpgjM6Ftb2BQdfjNj+CUnJHpxc CfFtM8lmeVrjPFHqtyuofVEqQvHjKSKSfZH5LrYA6QJfL0Girfqm3x4fgCSAaHPQ 87G7pcu5Ci2FDpQlHxPwnXHfbT/IKPNQrjB/nQ0E0ukBK5GRwm8AnqjK+cCcSaYo bcoNartPm8J3qPTC3Vdzw2OButC6wnXYrsbE1rADZ6PVByeXj7OtSX01fykHwgFj nEu4HEIMNTVtprZQ8UnMV95REM8yRteiNPfl+osK3Ae/PyZ/0go7bhYgfJh3qnIH NDzThiCy3/4YoBErejnLVQdpVTf2OFHEDpohwz76/4GhXouOiWkrSpjBWIwMnYC+ F/ZEgXkUIja6l1K16ovNe03DiEb/dgZkF//BJsnIAzNqK8PzGawwxLkDrMKVVAjP TpdGSnwyDsjjFMYG4s7dWWo0miwPU5QLJIkqqyk2040YCV6rgebpfZ8RrIuZEERi uSFQtf+ghBWGJDuf49P+adf20HbNYFIi++TkG0fQ0RlcagRHYRahgytdQpUBz9GS afOnC9uPtIyBq6dNLsvAr1pIlBXsy14lSdAv67rtUtOzNoeww0BaiYpL6Ir3rrPg pQ34U3RZ44kH0QWq/9btVWPTxLobRc41pf/LmEo4MAvwwZeyvJH5mJQotaWDRtJu NYLzCYcqT5UqRn5dYq8k8UFzLwaynoSxdUSV0uozGsLj5yXf5Dm1x/L+FAX5STgZ 95MntgS5fPjYABNPD9oVqfFdJmm9s27b3LgFgQBlx4kU+rqN/aAQdIltlUsA+zl1 qkoRdhilQfqn7g35HRm/ysD9pK7GKQ+R1D+fQAebs451E50q3VIaB0lJRluRXabK vWhMk29emRPKg8OMRR7r9oIss9gg+fTVmyttNX5MZ5t+DKjGdlApMhLPahyxa/i/ WWhRmZTLkVRqssD3blfPa1AR/XJIONcfFROwLyHy3LwQyMBcH8YcFC5oT3oDn3PR 07gXHSK6N7Xv6R08NF1Q6Ag4d6IUT//KvUf5WqyZaB5adLEE/Fip8doDed6zv/Hw qSeSqgWm9Y2t+6sekZXJkMHaY0Mdiuu6ONntG7cs87rHHKG/paCcAMLLoKkpFuJM voKkjlafMq+HR59hY4fIgrOw2iydL1Ihb9YiMzl15zknpzQeQMNCElm3dZvWwsHU n3HYTc+JqTEXxpP7IiBpIbiy/EP5kIuKk9kapBgiDDzMiey3WbyyeqTH4luAh0Uv ofJivNiEJeCzCR6Gml1bgvRbJrimsgYF1iBAsY3m14onkCCnWaim9kqLUmB8mVec xsYUJuNtGEqS8W0++uBzW+tXKlq6hjdGkfeCp/oD6gbBGxq+Iho8qVRSRPJ90EOR IZX5mCJJzSYdP9ELwWFAObl7WtRea7Ed376B6a7maQ== =EKUR -----END PGP MESSAGE----- --=-hf6zgIn8mtv3IPX/X7Tq-- ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/fixtures/mail/pgp_signed_1.eml ================================================ Message-ID: <1492765618.4984.1.camel@gandi.net> Subject: signed content From: Chamal To: Caliopen Date: Fri, 21 Apr 2017 11:06:58 +0200 In-Reply-To: <5B952AE1-6C33-42C5-B25C-AF4F7AD903D4@toto.net> Content-Type: multipart/signed; micalg="pgp-sha256"; protocol="application/pgp-signature"; boundary="=-pqHfzN3tdfFfcK4qJTZs" X-Mailer: Evolution 3.22.6 Mime-Version: 1.0 X-Evolution-Identity: 1485870052.1665.0@tao X-Evolution-Fcc: folder://1485870052.1665.2%40tao/Sent X-Evolution-Transport: 1485870052.1665.10@tao X-Evolution-Source-Folder: folder://1485870052.1665.2%40tao/INBOX X-Evolution-Source-Message: 2188502 X-Evolution-Source-Flags: ANSWERED ANSWERED_ALL SEEN X-Evolution-Source: --=-pqHfzN3tdfFfcK4qJTZs Content-Type: text/plain; charset="UTF-8" Content-Transfer-Encoding: quoted-printable Some clear text --=-pqHfzN3tdfFfcK4qJTZs Content-Type: application/pgp-signature; name="signature.asc" Content-Description: This is a digitally signed message part Content-Transfer-Encoding: 7bit -----BEGIN PGP SIGNATURE----- iQEzBAABCAAdFiEEOhFyGH73QG/Lzs+mDmjE+F0E2PwFAlj5y7IACgkQDmjE+F0E 2PycuggAjVQBUkJ1LzZyhKTyRilv/Wa14hYTg0eFlCRbeuo7WRWPB3/1EIo9CcF5 vL2r78Uwg4YUiXimjW02lfEqzPqBYWMm91dOwlN1PeDD6wuY0aWawJwXl8ow8z7S GP4N2GltfL+SXqPK6zOIvC2HDhyVVh+aIMuRpqt8S8senTWLTonsef0vUmzgjbVY z+cxKrcLCX1/rrygnj9J8j5gFkKeiB13/wAvqtsvN+HTkr2Uo0zsAQEjI+33VMCn V6HvalZobX1IuMJX+c6UVI0ZoC9Xy9isD6aMOg6ZGQbCkc+0KkFapkSnBxBtVHpq lBfSJH5YveSpj8ZmmeBirfGcnHhBaA== =/Kr9 -----END PGP SIGNATURE----- --=-pqHfzN3tdfFfcK4qJTZs-- ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/fixtures/vcard/multi.vcf ================================================ BEGIN:VCARD VERSION:3.0 FN:Patrice Tran N:Tran;Patrice;;; X-EVOLUTION-FILE-AS:Tran\, Patrice EMAIL;TYPE=OTHER:patrice@caliopen.org UID:pas-id-58AF0A5A00000000 REV:2017-02-23T16:14:18Z(1) END:VCARD BEGIN:VCARD VERSION:3.0 FN:Lison Ferez N:Ferez;Lison;;; X-EVOLUTION-FILE-AS:Ferez\, Lison EMAIL;TYPE=OTHER:lison@caliopen.org UID:pas-id-58AF0A5A00000001 REV:2017-02-23T16:14:18Z(1) END:VCARD ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/fixtures/vcard/rfc2425-1.vcard ================================================ cn:Babs Jensen cn:Barbara J Jensen sn:Jensen email:babs@umich.edu phone:+1 313 747-4454 x-id:1234567890 ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/fixtures/vcard/rfc2425-2.vcard ================================================ begin:VCARD source:ldap://cn=bjorn%20Jensen, o=university%20of%20Michigan, c=US name:Bjorn Jensen fn:Bj=F8rn Jensen n:Jensen;Bj=F8rn email;type=internet:bjorn@umich.edu tel;type=work,voice,msg:+1 313 747-4454 key;type=x509;encoding=B:dGhpcyBjb3VsZCBiZSAKbXkgY2VydGlmaWNhdGUK end:VCARD ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/fixtures/vcard/rfc2425-3.vcard ================================================ begin:vcard source:ldap://cn=Meister%20Berger,o=Universitaet%20Goerlitz,c=DE name:Meister Berger fn:Meister Berger n:Berger;Meister bday;value=date:1963-09-21 o:Universit=E6t G=F6rlitz title:Mayor title;language=de;value=text:Burgermeister note:The Mayor of the great city of Goerlitz in the great country of Germany. email;internet:mb@goerlitz.de home.tel;type=fax,voice,msg:+49 3581 123456 home.label:Hufenshlagel 1234\n 02828 Goerlitz\n Deutschland key;type=X509;encoding=b:MIICajCCAdOgAwIBAgICBEUwDQYJKoZIhvcNAQEEBQ AwdzELMAkGA1UEBhMCVVMxLDAqBgNVBAoTI05ldHNjYXBlIENvbW11bmljYXRpb25zI ENvcnBvcmF0aW9uMRwwGgYDVQQLExNJbmZvcm1hdGlvbiBTeXN0ZW1zMRwwGgYDVQQD ExNyb290Y2EubmV0c2NhcGUuY29tMB4XDTk3MDYwNjE5NDc1OVoXDTk3MTIwMzE5NDc 1OVowgYkxCzAJBgNVBAYTAlVTMSYwJAYDVQQKEx1OZXRzY2FwZSBDb21tdW5pY2F0aW 9ucyBDb3JwLjEYMBYGA1UEAxMPVGltb3RoeSBBIEhvd2VzMSEwHwYJKoZIhvcNAQkBF hJob3dlc0BuZXRzY2FwZS5jb20xFTATBgoJkiaJk/IsZAEBEwVob3dlczBcMA0GCSqG SIb3DQEBAQUAA0sAMEgCQQC0JZf6wkg8pLMXHHCUvMfL5H6zjSk4vTTXZpYyrdN2dXc oX49LKiOmgeJSzoiFKHtLOIboyludF90CgqcxtwKnAgMBAAGjNjA0MBEGCWCGSAGG+E IBAQQEAwIAoDAfBgNVHSMEGDAWgBT84FToB/GV3jr3mcau+hUMbsQukjANBgkqhkiG9 w0BAQQFAAOBgQBexv7o7mi3PLXadkmNP9LcIPmx93HGp0Kgyx1jIVMyNgsemeAwBM+M SlhMfcpbTrONwNjZYW8vJDSoi//yrZlVt9bJbs7MNYZVsyF1unsqaln4/vy6Uawfg8V UMk1U7jt8LYpo4YULU7UZHPYVUaSgVttImOHZIKi4hlPXBOhcUQ== end:vcard ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/fixtures/vcard/rfc2426-1.vcard ================================================ BEGIN:VCARD FN:Rene van der Harten N:van der Harten;Rene;J.;Sir;R.D.O.N. SORT-STRING:Harten END:VCARD ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/fixtures/vcard/rfc2426-2.vcard ================================================ BEGIN:VCARD FN:Robert Pau Shou Chang N:Pau;Shou Chang;Robert SORT-STRING:Pau END:VCARD ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/fixtures/vcard/rfc2426-3.vcard ================================================ BEGIN:VCARD FN:Osamu Koura N:Koura;Osamu SORT-STRING:Koura END:VCARD ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/fixtures/vcard/rfc2426-4.vcard ================================================ BEGIN:VCARD FN:Oscar del Pozo N:del Pozo Triscon;Oscar SORT-STRING:Pozo END:VCARD ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/fixtures/vcard/rfc2426-5.vcard ================================================ BEGIN:VCARD FN:Chistine d'Aboville N:d'Aboville;Christine SORT-STRING:Aboville END:VCARD ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/fixtures/vcard/vcard1.vcf ================================================ BEGIN:VCARD VERSION:2.1 ADR;TYPE=WORK:;;63-65 Boulevard Massena;Paris;;75314;France ADR;TYPE=WORK:;;63-65 Boulevard Massena;Paris;;75314;France EMAIL;TYPE=WORK:jeffrey1@osafoundation.org EMAIL:jeffrey2@osafoundation.org EMAIL:jeffrey3@osafoundation.org N:Harris;Jeffrey;;; IMPP:aim:johndoe@aol.com NOTE;ENCODING=QUOTED-PRINTABLE: This is a note associated with this TEL:+33 1 02 03 04 05 TEL:+33 (0)1 02 03 04 05 FN:Jeffrey Harris KEY;TYPE=PGP:http://pgp.mit.edu:11371/pks/lookup?op=get&search=0x9F0FE587374BBE81 KEY;PGP:http://example.com/key.pgp CATEGORIES:swimmer,biker END:VCARD ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/fixtures/vcard/vcard2.vcf ================================================ BEGIN:VCARD BDAY;VALUE=DATE:1963-09-21 VERSION:3.0 ADR;TYPE=WORK,POSTAL,PARCEL:;; EMAIL;TYPE=INTERNET:deriks@Microsoft.com N:Stenerson;Derik IMPP:aim:johndoe@aol.com NOTE:I am proficient in Tiger-Crane Style,\nand I am more than proficient in the exquisite art of the Samurai sword. TEL:+343 1 02 03 04 05 TEL:+343 (0)1 02 03 04 05 TEL:+355 1832 3849 TEL:+61 777 888 999 TEL:+43 1 765 437 TEL:+32 33 12 34 56 TEL:+229 90 89 77 66 FN:Derik Stenerson KEY;TYPE=PGP:http://example.com/key.pgp ORG:Microsoft Corporation CATEGORIES:swimmer,biker END:VCARD ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/fixtures/vcard/vcard3.vcf ================================================ BEGIN:VCARD VERSION:4.0 ADR;TYPE=WORK:;;100 Waters Edge;Baytown;LA;30314;United States of America ADR;TYPE=home:;;123 Main St.;Springfield;IL;12345;USA EMAIL;TYPE=PREF,INTERNET:forrestgump@example.com TEL:+93 77 888 9999 TEL:+852.23.21.33.34 TEL:+389 4 345 4456 TEL:+52 (13) 33 22 44 55 TEL:+235 [22/66/99/77]902876 TEL:+58 234 454 55 67 N:Stevenson;John;Philip,Paul;Dr.;Jr.,M.D.,A.C.P. IMPP:aim:johndoe@aol.com IMPP;TYPE=personal,text,store,pref:im:john@example.com NOTE:I am proficient in Tiger-Crane Style,\nand I am more than proficient in the exquisite art of the Samurai sword. FN:Forrest Gump FN:Dr. John Doe ORG:Bubba Gump Shrimp Co. ORG:ABC\, Inc.;North American Division;Marketing TITLE:Shrimp Man LABEL;TYPE=HOME:42 Plantation St.\nBaytown, LA 30314\nUnited States of America REV:20080424T195243Z KEY:data:application/x-pgp-fingerprint,5E61C8780F86295CE17D86779F0FE587374BBE81 KEY:http://pgp.mit.edu:11371/pks/lookup?op=get&search=0x9F0FE587374BBE81 KEY:MEDIATYPE=application/pgp-keys:http://example.com/key.pgp CATEGORIES:swimmer,biker PHOTO;VALUE=URL;TYPE=GIF:http://www.example.com/dir_photos/my_photo.gif END:VCARD ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/parsers/__init__.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/parsers/test_email.py ================================================ import unittest import os from datetime import datetime from zope.interface.verify import verifyObject from caliopen_storage.config import Configuration import vobject if 'CALIOPEN_BASEDIR' in os.environ: conf_file = '{}/src/backend/configs/caliopen.yaml.template'. \ format(os.environ['CALIOPEN_BASEDIR']) else: conf_file = '../../../../../configs/caliopen.yaml.template' Configuration.load(conf_file, 'global') from caliopen_main.common.helpers.normalize import clean_email_address class TestEmailParser(unittest.TestCase): def test_simple_1(self): email = 'toto@toto.fr' res = clean_email_address(email) self.assertEqual(email, res[0]) self.assertEqual(email, res[1]) def test_with_name_1(self): email = '"Ceci est, une virgule" ' res = clean_email_address(email) self.assertEqual('test@toto.com', res[0]) self.assertEqual('test@toto.com', res[1]) def test_multiple(self): emails = '"Ceci est, une virgule" , ' \ '"Est une, autre virgule" ' parts = emails.split('>,') self.assertEqual(len(parts), 2) for part in parts: res = clean_email_address(part) self.assertTrue('@' in res[0]) def test_invalid_but_valid(self): email = 'Ceci [lamentable.ment] ' res = clean_email_address(email) self.assertEqual('email@truc.tld', res[0]) def test_strange_1(self): email = 'ideascube/ideascube ' res = clean_email_address(email) self.assertEqual('ideascube@noreply.github.com', res[0]) ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/parsers/test_mail.py ================================================ """Test mail message format processing.""" import unittest import os from datetime import datetime from zope.interface.verify import verifyObject from caliopen_storage.config import Configuration if 'CALIOPEN_BASEDIR' in os.environ: conf_file = '{}/src/backend/configs/caliopen.yaml.template'. \ format(os.environ['CALIOPEN_BASEDIR']) else: conf_file = '../../../../../configs/caliopen.yaml.template' Configuration.load(conf_file, 'global') from caliopen_main.common.interfaces import IMessageParser from caliopen_main.message.parsers.mail import MailMessage def load_mail(filename): """Read email from fixtures of an user.""" # XXX tofix: set fixtures in a more convenient way to not # have dirty hacking on relative path dir_path = os.path.dirname(os.path.realpath(__file__)) path = '{}/../fixtures/mail'.format(dir_path) with open('{}/{}'.format(path, filename)) as f: data = f.read() return data class TestMailFormat(unittest.TestCase): """Test formatting of a mail objet (rfc2822).""" def test_signed_mail(self): """Test parsing of a pgp signed mail.""" data = load_mail('pgp_signed_1.eml') mail = MailMessage(data) self.assertTrue(verifyObject(IMessageParser, mail)) self.assertTrue(len(mail.participants) > 1) self.assertEqual(len(mail.attachments), 1) self.assertEqual(mail.subject, 'signed content') self.assertTrue(isinstance(mail.date, datetime)) def test_encrypted_mail(self): """Test parsing of a pgp encrypted mail.""" data = load_mail('pgp_crypted_1.eml') mail = MailMessage(data) self.assertTrue(verifyObject(IMessageParser, mail)) self.assertTrue(len(mail.participants) > 1) self.assertEqual(len(mail.attachments), 1) self.assertEqual(mail.subject, 'crypted content') self.assertTrue(isinstance(mail.date, datetime)) self.assertTrue(mail.extra_parameters.get('encrypted', None), 'pgp') if __name__ == '__main__': unittest.main() ================================================ FILE: src/backend/main/py.main/caliopen_main/tests/parsers/test_vcard.py ================================================ import unittest import os from datetime import datetime from zope.interface.verify import verifyObject from caliopen_storage.config import Configuration import vobject if 'CALIOPEN_BASEDIR' in os.environ: conf_file = '{}/src/backend/configs/caliopen.yaml.template'. \ format(os.environ['CALIOPEN_BASEDIR']) else: conf_file = '../../../../../configs/caliopen.yaml.template' Configuration.load(conf_file, 'global') #from caliopen_main.interfaces import IMessageParser from caliopen_main.contact.parameters import NewContact from caliopen_main.contact.parsers import VcardContact def load_vcard(filename): dir_path = os.path.dirname(os.path.realpath(__file__)) path = '{}/../fixtures/vcard'.format(dir_path) with open('{}/{}'.format(path, filename)) as f: data = f.read() return data def parse_vcard(vcard): contact = VcardContact(vcard) return contact.contact class TestVcardFormat(unittest.TestCase): def test_name_vcard(self): data = load_vcard('vcard2.vcf') vcard = vobject.readOne(data) contact = parse_vcard(vcard) self.assertIsNotNone(contact.family_name) self.assertIsNotNone(contact.given_name) def test_address_vcard(self): data = load_vcard('vcard1.vcf') vcard = vobject.readOne(data) contact = parse_vcard(vcard) for i in contact.addresses: self.assertIsNotNone(i.city) def test_email_vcard(self): data = load_vcard('vcard1.vcf') vcard = vobject.readOne(data) contact = parse_vcard(vcard) for i in contact.emails: self.assertIsNotNone(i.address) def test_ims_vcard(self): data = load_vcard('vcard1.vcf') vcard = vobject.readOne(data) contact = parse_vcard(vcard) for i in contact.ims: self.assertIsNotNone(i.type) if __name__ == '__main__': unittest.main() ================================================ FILE: src/backend/main/py.main/caliopen_main/user/__init__.py ================================================ # -*- coding: utf-8 -*- ================================================ FILE: src/backend/main/py.main/caliopen_main/user/core/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .user import User, Tag, FilterRule, UserIdentity, ReservedName from .user import allocate_user_shard from .identity import IdentityLookup, IdentityTypeLookup __all__ = [ 'User', 'Tag', 'FilterRule', 'UserIdentity', 'ReservedName', 'allocate_user_shard', 'IdentityLookup', 'IdentityTypeLookup' ] ================================================ FILE: src/backend/main/py.main/caliopen_main/user/core/identity.py ================================================ # -*- coding: utf-8 -*- """Caliopen user related core classes.""" from __future__ import absolute_import, print_function, unicode_literals from caliopen_storage.core import BaseCore from caliopen_main.common.core import BaseUserCore from ..store import (UserIdentity as ModelUserIdentity, IdentityLookup as ModelIdentityLookup, IdentityTypeLookup as ModelIdentityTypeLookup) class UserIdentity(BaseUserCore): """User's identity core class.""" _model_class = ModelUserIdentity _pkey_name = 'identity_id' @classmethod def get_by_identifier(cls, identifier, protocol, user_id): """return an array of one or more user_identities""" if not protocol: ids = IdentityLookup.find(identifier=identifier) elif not user_id: ids = IdentityLookup.find(identifier=identifier, protocol=protocol) else: ids = IdentityLookup.find(identifier=identifier, protocol=protocol, user_id=user_id) identities = [] for id in ids: identities.append( UserIdentity.get_by_user_id(id.user_id, id.identity_id)) return identities class IdentityLookup(BaseCore): """Lookup table core class.""" _model_class = ModelIdentityLookup _pkey_name = 'identifier' class IdentityTypeLookup(BaseCore): """Lookup table core class""" _model_class = ModelIdentityTypeLookup _pkey_name = 'type' ================================================ FILE: src/backend/main/py.main/caliopen_main/user/core/setups.py ================================================ # -*- coding: utf-8 -*- """new user related logic""" from __future__ import absolute_import, print_function, unicode_literals import logging import datetime import pytz from caliopen_storage.config import Configuration from elasticsearch import Elasticsearch from caliopen_storage.core import core_registry from caliopen_main.user.objects.settings import Settings as ObjectSettings log = logging.getLogger(__name__) def setup_index(user): """Creates user index and setups mappings.""" url = Configuration('global').get('elasticsearch.url') client = Elasticsearch(url) shard_id = user.shard_id # does shard exist ? if not client.indices.exists(shard_id): setup_shard_index(shard_id) index_name = shard_id alias_name = user.user_id # Points an alias to the underlying user's index try: client.indices.put_alias(index=index_name, name=alias_name) except Exception as exc: log.exception("failed to create alias : {0}".format(exc)) raise exc def setup_shard_index(shard): """Setup a shard index.""" url = Configuration('global').get('elasticsearch.url') client = Elasticsearch(url) try: log.info('Creating index {0}'.format(shard)) client.indices.create( index=shard, body={ "settings": { "analysis": { "analyzer": { "text_analyzer": { "type": "custom", "tokenizer": "lowercase", "filter": [ "ascii_folding" ] }, "email_analyzer": { "type": "custom", "tokenizer": "email_tokenizer", "filter": [ "ascii_folding" ] } }, "filter": { "ascii_folding": { "type": "asciifolding", "preserve_original": True } }, "tokenizer": { "email_tokenizer": { "type": "ngram", "min_gram": 3, "max_gram": 25 } } } } }) except Exception as exc: log.warn("failed to create index {} : {}".format(shard, exc)) return # TOFIX # core Message is no more in core_registry, use hard coded build mapping from caliopen_main.message.store.message_index import IndexedMessage from caliopen_main.contact.store.contact_index import IndexedContact log.info('Creating index mapping for message and contact in shard {}'. format(shard)) message_mapping = IndexedMessage.build_mapping() message_mapping.save(shard, using=client) contact_mapping = IndexedContact.build_mapping() contact_mapping.save(shard, using=client) def setup_system_tags(user): """Create system tags.""" # TODO: translate tags'name to user's preferred language default_tags = Configuration('global').get('system.default_tags') for tag in default_tags: tag['type'] = 'system' tag['date_insert'] = datetime.datetime.now(tz=pytz.utc) tag['label'] = tag.get('label', tag['name']) from .user import Tag Tag.create(user, **tag) def setup_settings(user, settings): """Create settings related to user.""" # XXX set correct values settings = { 'user_id': user.user_id, 'default_locale': settings.default_locale, 'message_display_format': settings.message_display_format, 'contact_display_order': settings.contact_display_order, 'contact_display_format': settings.contact_display_format, 'notification_enabled': settings.notification_enabled, 'notification_message_preview': settings.notification_message_preview, 'notification_sound_enabled': settings.notification_sound_enabled, 'notification_delay_disappear': settings.notification_delay_disappear, } obj = ObjectSettings(user) obj.unmarshall_dict(settings) obj.marshall_db() obj.save_db() return True ================================================ FILE: src/backend/main/py.main/caliopen_main/user/core/user.py ================================================ # -*- coding: utf-8 -*- """Caliopen user related core classes.""" from __future__ import absolute_import, print_function, unicode_literals import os import datetime import pytz import bcrypt import logging import uuid from zxcvbn import zxcvbn from validate_email import validate_email from caliopen_storage.config import Configuration from caliopen_storage.exception import NotFound, CredentialException from ..store import (User as ModelUser, UserName as ModelUserName, UserRecoveryEmail as ModelUserRecoveryEmail, IndexUser, UserTag as ModelUserTag, Settings as ModelSettings, FilterRule as ModelFilterRule, ReservedName as ModelReservedName) from ..core.identity import UserIdentity, IdentityLookup, IdentityTypeLookup from caliopen_storage.core import BaseCore from caliopen_main.common.core import BaseUserCore from caliopen_main.contact.core import Contact as CoreContact from caliopen_main.contact.parameters import NewEmail from caliopen_main.pi.objects import PIModel from caliopen_main.user.helpers import validators from .setups import (setup_index, setup_system_tags, setup_settings) log = logging.getLogger(__name__) def allocate_user_shard(user_id): """Find allocation to a shard for an user.""" shards = Configuration('global').get('elasticsearch.shards') if not shards: raise Exception('No shards configured for index') shard_idx = int(user_id.hex, 16) % len(shards) return shards[shard_idx] class Tag(BaseUserCore): """Tag core object.""" _model_class = ModelUserTag _pkey_name = 'tag_id' class FilterRule(BaseUserCore): """Filter rule core class.""" _model_class = ModelFilterRule _pkey_name = 'rule_id' @classmethod def create(cls, user, rule): """Create a new filtering rule.""" rule.validate() # XXX : expr value is evil o = super(FilterRule, cls).create(user_id=user.user_id, rule_id=user.new_rule_id(), date_insert=datetime.datetime.now( tz=pytz.utc), name=rule.name, filter_expr=rule.expr, position=rule.position, stop_condition=rule.stop_condition, tags=rule.tags) return o def eval(self, message): """ Evaluate if this rule apply to the given message. evaluation return a list of TAGS to add or nothing stop condition if set and if result is empty or not and match this stop condition, processing of rules on this message stop. """ # XXXX WARN WARN WARN WARN WARN WARN WARN WARN # # This is a REALLY BASIC filtering concept with no # security consideration abount what is evaluated !!!! # # XXXX WARN WARN WARN WARN WARN WARN WARN WARN res = eval(self.filter_expr) if self.stop_condition is not None: if self.stop_condition and res: return self.tags, True if not self.stop_condition and not res: return self.tags, True if res: return self.tags, False return [], False class ReservedName(BaseCore): """Reserved name core object.""" _model_class = ModelReservedName _pkey_name = 'name' class UserName(BaseCore): """User name core object.""" _model_class = ModelUserName _pkey_name = 'name' class UserRecoveryEmail(BaseCore): """User Recovery Email object to retrieve user by recovery_email""" _model_class = ModelUserRecoveryEmail _pkey_name = 'recovery_email' class Settings(BaseUserCore): """User settings core object.""" # XXX this core object is here to fill core_registry # it's not used, objects representation have to be used. _model_class = ModelSettings _pkey_name = None class User(BaseCore): """User core object.""" _model_class = ModelUser _pkey_name = 'user_id' _index_class = IndexUser @classmethod def _check_whitelistes(cls, user): """Check if user is in a white list if apply.""" whitelistes = Configuration('global').get('whitelistes', {}) emails_file = whitelistes.get('user_emails') if emails_file and os.path.isfile(emails_file): with open(emails_file) as f: emails = [x for x in f.read().split('\n') if x] if user.recovery_email in emails: return True else: raise ValueError('user email not in whitelist') @classmethod def _check_max_users(cls): """Check if maximum number of users reached.""" conf = Configuration('global').get('system', {}) max_users = conf.get('max_users', 0) if max_users: nb_users = User._model_class.objects.count() if nb_users >= max_users: raise ValueError('Max number of users reached') @classmethod def create(cls, new_user): """Create a new user. @param: new_user is a parameters/user.py.NewUser object # 1.check username regex # 2.check username is not in reserved_name table # 3.check recovery email validity (TODO : check if email is not within # the current Caliopen's instance) # 4.check username availability # 5.add username to user cassa user_name table (to block the # availability) # 6.check password strength (and regex?) # then # create user and linked contact """ def rollback_username_storage(username): UserName.get(username).delete() # 0. check for user email white list and max number of users cls._check_whitelistes(new_user) cls._check_max_users() # 1. try: validators.is_valid_username(new_user.name) except SyntaxError: raise ValueError("Malformed username") # 2. try: ReservedName.get(new_user.name) raise ValueError('Reserved user name') except NotFound: pass user_id = uuid.uuid4() # 3. if not new_user.recovery_email: raise ValueError("Missing recovery email") try: cls.validate_recovery_email(new_user.recovery_email) except Exception as exc: log.info("recovery email failed validation : {}".format(exc)) raise ValueError(exc) # 4. & 5. if User.is_username_available(new_user.name.lower()): # save username immediately to prevent concurrent creation UserName.create(name=new_user.name.lower(), user_id=user_id) # NB : need to rollback this username creation if the below # User creation failed for any reason else: raise ValueError("Username already exist") # 6. try: user_inputs = [new_user.name.encode("utf-8"), new_user.recovery_email.encode("utf-8")] # TODO: add contact inputs if any password_strength = zxcvbn(new_user.password, user_inputs=user_inputs) privacy_features = {"password_strength": str(password_strength["score"])} passwd = new_user.password.encode('utf-8') new_user.password = bcrypt.hashpw(passwd, bcrypt.gensalt()) except Exception as exc: log.exception(exc) rollback_username_storage(new_user.name) raise exc try: new_user.validate() # schematic model validation except Exception as exc: rollback_username_storage(new_user.name) log.info("schematics validation error: {}".format(exc)) raise ValueError("new user malformed") try: recovery = new_user.recovery_email if hasattr(new_user, "contact"): family_name = new_user.contact.family_name given_name = new_user.contact.given_name else: family_name = "" given_name = "" # XXX PI compute pi = PIModel() pi.technic = 0 pi.comportment = 0 pi.context = 0 pi.version = 0 shard_id = allocate_user_shard(user_id) core = super(User, cls).create(user_id=user_id, name=new_user.name, password=new_user.password, recovery_email=recovery, params=new_user.params, date_insert=datetime.datetime.now( tz=pytz.utc), privacy_features=privacy_features, pi=pi, family_name=family_name, given_name=given_name, shard_id=shard_id) except Exception as exc: log.info(exc) rollback_username_storage(new_user.name) raise exc # **** operations below do not raise fatal error and rollback **** # # Setup index setup_index(core) # Setup others entities related to user setup_system_tags(core) setup_settings(core, new_user.settings) UserRecoveryEmail.create(recovery_email=recovery, user_id=user_id) # Add a default local identity on a default configured domain default_domain = Configuration('global').get('default_domain') default_local_id = '{}@{}'.format(core.name, default_domain) if not core.add_local_identity(default_local_id): log.warn('Impossible to create default local identity {}'. format(default_local_id)) # save and index linked contact if hasattr(new_user, "contact"): #  add local email to contact local_mail = NewEmail() local_mail.address = default_local_id new_user.contact.emails.append(local_mail) # create default contact for user contact = CoreContact.create(core, new_user.contact) core.model.contact_id = contact.contact_id log.info("contact id {} for new user {}".format(contact.contact_id, core.user_id)) else: log.error( "missing contact in new_user params for user {}. " "Can't create related tables".format(core.user_id)) core.save() return core @classmethod def by_name(cls, name): """Get user by name.""" uname = UserName.get(name.lower()) return cls.get(uname.user_id) @classmethod def is_username_available(cls, username): """Return True if username is available.""" try: UserName.get(username.lower()) return False except NotFound: return True @classmethod def by_local_identifier(cls, address, protocol): """Get a user by one of a local identifier.""" identities = UserIdentity.get_by_identifier(address.lower(), protocol, None) for identity in identities: if identity.type == 'local': return cls.get(identity.user_id) raise NotFound @classmethod def authenticate(cls, user_name, password): """Authenticate an user.""" try: user = cls.by_name(user_name) except NotFound: raise CredentialException('Invalid user') if user.date_delete: raise CredentialException('Invalid credentials') # XXX : decode unicode not this way if bcrypt.hashpw(str(password.encode('utf-8')), str(user.password)) == user.password: return user raise CredentialException('Invalid credentials') @property def contact(self): """User is a contact.""" if self.contact_id is None: return None try: return CoreContact.get(self, self.contact_id) except NotFound: log.warn("contact {} not found for user {}". format(self.contact_id, self.user_id)) return None @property def tags(self): """Tag objects related to an user.""" objs = Tag._model_class.filter(user_id=self.user_id) return [Tag(x) for x in objs] @property def rules(self): """Filtering rules associated to an user, sorted by position.""" objs = FilterRule._model_class.filter(user_id=self.user_id) cores = [FilterRule(x) for x in objs] return sorted(cores, key=lambda x: x.position) def add_local_identity(self, address): """ Add a local smtp identity to an user and fill related lookup tables """ formatted = address.lower() try: local_identity = User.by_local_identifier(address, 'smtp') if local_identity: log.warn("local identifier {} already exist".format(address)) return None except NotFound: try: identities = IdentityLookup.find(identifier=formatted) if len(identities) == 0: raise NotFound for id in identities: identity = UserIdentity.get(self, id.identity_id) if identity and identity.protocol == 'smtp' and \ identity.user_id == self.user_id: if identity.identity_id in self.local_identities: # Local identity already created. Should raise error ? return identity raise Exception( 'Inconsistent local identity {}'.format(address)) except NotFound: display_name = "{} {}".format(self.given_name, self.family_name) identity = UserIdentity.create(self, identifier=formatted, identity_id=uuid.uuid4(), type='local', status='active', protocol='email', display_name=display_name) #  insert entries in relevant lookup tables IdentityLookup.create(identifier=identity.identifier, protocol=identity.protocol, user_id=identity.user_id, identity_id=identity.identity_id) IdentityTypeLookup.create(type=identity.type, user_id=identity.user_id, identity_id=identity.identity_id) return identity except Exception as exc: log.error('Unexpected exception {}'.format(exc)) return None @property def local_identities(self): return IdentityTypeLookup.find(type='local', user_id=self.user_id) @classmethod def validate_recovery_email(cls, email): """ provided email has to pass the validations below, otherwise this func will raise an error @:arg email: string """ # 1. is email well-formed ? if not validate_email(email): raise ValueError("recovery email malformed") # 2. is email already in our db ? (recovery_email must be unique) try: UserRecoveryEmail.get(email) except NotFound: pass else: raise ValueError("recovery email already used in this instance") # 3. is email belonging to one of our domains ? # (recovery_email must be external) domain = email.split("@")[1] if domain in Configuration("global").get("default_domain"): raise ValueError( "recovery email must be outside of this domain instance") # 4. TODO: check that provided recovery email can really receive email # send a confirmation email ? ================================================ FILE: src/backend/main/py.main/caliopen_main/user/helpers/__init__.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/user/helpers/mergePatch.py ================================================ # -*- coding: utf-8 -*- """ Functions to handle patch dict arriving from clients to update client's resources """ import logging log = logging.getLogger(__name__) def merge_patch(target, patch): """rfc 7396 merge patch implementation""" if type(patch) is dict: if type(target) is not dict: target = {} # Ignore the contents and set it to an empty Object for key, value in patch.iteritems(): if value is None: if key in target: del target[key] else: target[key] = merge_patch(target[key], value) return target else: return patch class MergeBatch(object): """Describe a batch of operations for processing a merge.""" def __init__(self, core): self.obj = core self.operations = [] def add_operation(self, type, column, value): self.operations.append((type, column, value)) def process(self): """Process the merge on core object.""" log.info("process operations : {}".format(self.operations)) for typ, column, value in self.operations: core_attr = getattr(self.obj, column) if typ == 'add': core_attr = core_attr + value elif typ == 'del': core_attr = core_attr - value elif typ == 'replace': core_attr = value else: raise NotImplementedError('Merge operation {} not supported', format(typ)) self.obj.save() ================================================ FILE: src/backend/main/py.main/caliopen_main/user/helpers/validators.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals import regex def is_valid_username(username): """ conforms to doc/RFCs/username_specifications""" if len(username) < 3: raise SyntaxError rgx = ur"^(([^\p{C}\p{M}\p{Lm}\p{Sk}\p{Z}.\u0022,@\u0060:;<>\[\\\]]|" \ ur"[^\p{C}\p{M}\p{Lm}\p{Sk}\p{Z}.\u0022,@\u0060:;<>\[\\\]]\.)){1,40}" \ ur"[^\p{C}\p{M}\p{Lm}\p{Sk}\p{Z}.\u0022,@\u0060:;<>\[\\\]]$" r = regex.compile(rgx, flags=regex.V1+regex.UNICODE) if r.match(username) is None: raise SyntaxError def main(): try: is_valid_username("0123❤xxx___the______johnΔœuf") except: print("username NOT valid") return print("username OK") if __name__ == '__main__': main() ================================================ FILE: src/backend/main/py.main/caliopen_main/user/objects/__init__.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/user/objects/device.py ================================================ ================================================ FILE: src/backend/main/py.main/caliopen_main/user/objects/identity.py ================================================ # -*- coding: utf-8 -*- """Caliopen message object classes.""" from __future__ import absolute_import, print_function, unicode_literals import types import datetime from uuid import UUID from caliopen_main.common.objects.base import ObjectUser from ..store.identity import UserIdentity as ModelUserIdentity class Credentials(): _attrs = {} class UserIdentity(ObjectUser): """Local or remote identity related to an user.""" _attrs = { 'credentials': Credentials, 'display_name': types.StringType, 'identifier': types.StringType, # for example: me@caliopen.org 'identity_id': UUID, 'infos': types.DictionaryType, 'last_check': datetime.datetime, 'protocol': types.StringType, # for example: smtp, imap, mastodon 'status': types.StringType, # for example : active, inactive, deleted 'type': types.StringType, # for example : local, remote 'user_id': UUID } _model_class = ModelUserIdentity _pkey_name = 'identity_id' _db = None # model instance with data from db ================================================ FILE: src/backend/main/py.main/caliopen_main/user/objects/settings.py ================================================ # -*- coding: utf-8 -*- """Caliopen User tag parameters classes.""" from __future__ import absolute_import, print_function, unicode_literals import logging import types import uuid from caliopen_main.user.parameters.settings import Settings as SettingsParam from caliopen_main.user.store import Settings as ModelSettings from caliopen_main.common.objects.base import ObjectUser log = logging.getLogger(__name__) class Settings(ObjectUser): """Settings related to an user.""" _attrs = { 'user_id': uuid.UUID, 'default_locale': types.StringType, 'message_display_format': types.StringType, 'contact_display_order': types.StringType, 'contact_display_format': types.StringType, 'notification_enabled': types.BooleanType, 'notification_message_preview': types.StringType, 'notification_sound_enabled': types.BooleanType, 'notification_delay_disappear': types.IntType, } _model_class = ModelSettings _pkey_name = None _json_model = SettingsParam ================================================ FILE: src/backend/main/py.main/caliopen_main/user/objects/tag.py ================================================ # -*- coding: utf-8 -*- """Caliopen User tag parameters classes.""" from __future__ import absolute_import, print_function, unicode_literals import types import uuid import datetime from caliopen_main.common.objects.base import ObjectUser from caliopen_main.user.store import UserTag as ModelUserTag import logging log = logging.getLogger(__name__) class UserTag(ObjectUser): """Tag related to an user.""" _attrs = { 'date_insert': datetime.datetime, 'importance_level': types.IntType, 'name': types.StringType, 'label': types.StringType, 'type': types.StringType, 'user_id': uuid.UUID } _model_class = ModelUserTag _pkey_name = 'tag_id' def delete_db(self): """Delete a tag in store.""" self._db.delete() return True ================================================ FILE: src/backend/main/py.main/caliopen_main/user/parameters/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .user import NewUser, User, NewRule from caliopen_main.common.parameters.types import InternetAddressType, PhoneNumberType from .tag import NewUserTag, UserTag from .identity import NewUserIdentity, UserIdentity from .settings import Settings __all__ = [ 'InternetAddressType', 'PhoneNumberType', 'NewUser', 'User', 'NewRule', 'NewUserTag', 'UserTag', 'NewUserIdentity', 'UserIdentity', 'Settings', ] ================================================ FILE: src/backend/main/py.main/caliopen_main/user/parameters/identity.py ================================================ # -*- coding: utf-8 -*- """Caliopen user parameters.""" from schematics.models import Model from schematics.types import StringType, UUIDType, DateTimeType from schematics.types.compound import DictType import caliopen_storage.helpers.json as helpers USER_IDENTITY_PROTOCOLS = ['smtp', 'imap'] USER_IDENTITY_STATUS = ['active', 'inactive', 'deleted'] USER_IDENTITY_TYPES = ['local', 'remote'] class NewUserIdentity(Model): credentials = DictType(StringType, default=lambda: {}) display_name = StringType() identifier = StringType(required=True) infos = DictType(StringType, default=lambda: {}) protocol = StringType(choices=USER_IDENTITY_PROTOCOLS) status = StringType(default='active', choices=USER_IDENTITY_STATUS) type = StringType(choices=USER_IDENTITY_TYPES) class UserIdentity(NewUserIdentity): user_id = UUIDType() identity_id = UUIDType() last_check = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') ================================================ FILE: src/backend/main/py.main/caliopen_main/user/parameters/settings.py ================================================ # -*- coding: utf-8 -*- """Caliopen contact parameters classes.""" from __future__ import absolute_import, print_function, unicode_literals from schematics.models import Model from schematics.types import StringType, IntType, BooleanType MESSAGE_FORMAT_CHOICES = ['rich_text', 'plain_text'] CONTACT_FORMAT_CHOICES = ['given_name, family_name', 'family_name, given_name'] CONTACT_ORDER_CHOICES = ['family_name', 'given_name'] PREVIEW_CHOICES = ['off', 'always'] DELAY_CHOICES = [0, 5, 10, 30] class Settings(Model): """Location structure for a device.""" default_locale = StringType(default='fr-FR') message_display_format = StringType(default='rich_text', choices=MESSAGE_FORMAT_CHOICES) contact_display_format = StringType(default='family_name, given_name', choices=CONTACT_FORMAT_CHOICES) contact_display_order = StringType(default='given_name', choices=CONTACT_ORDER_CHOICES) notification_enabled = BooleanType(default=True) notification_message_preview = StringType(default='always', choices=PREVIEW_CHOICES) notification_sound_enabled = BooleanType(default=False) notification_delay_disappear = IntType(default=10, choices=DELAY_CHOICES) ================================================ FILE: src/backend/main/py.main/caliopen_main/user/parameters/tag.py ================================================ # -*- coding: utf-8 -*- """Caliopen tags parameters.""" from schematics.models import Model from schematics.types import StringType, UUIDType, DateTimeType from schematics.transforms import blacklist import caliopen_storage.helpers.json as helpers class NewUserTag(Model): """Create a new user tag.""" user_id = UUIDType() name = StringType() class Option: roles = {'default': blacklist('user_id')} serialize_when_none = False class UserTag(NewUserTag): """Existing user tag.""" type = StringType() date_insert = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') class ImportedTag(Model): """Create a tag from external label or flag""" user_id = UUIDType() label = StringType() name = StringType() type = StringType() class Option: roles = {'default': blacklist('user_id')} serialize_when_none = False ================================================ FILE: src/backend/main/py.main/caliopen_main/user/parameters/user.py ================================================ # -*- coding: utf-8 -*- """Caliopen user parameters.""" from schematics.models import Model from schematics.types import (StringType, UUIDType, IntType, DateTimeType, BooleanType, EmailType) from schematics.types.compound import ModelType, DictType, ListType from schematics.transforms import blacklist from caliopen_main.contact.parameters import NewContact, Contact from .settings import Settings from caliopen_main.pi.parameters import PIParameter import caliopen_storage.helpers.json as helpers class NewUser(Model): """ Parameter to create a new user. name, recovery_email and password are required a ``NewContact`` can be attached when creating user """ contact = ModelType(NewContact) main_user_id = UUIDType() name = StringType(required=True) params = DictType(StringType()) password = StringType(required=True) recovery_email = EmailType(required=True) settings = ModelType(Settings, default=lambda: {}) class Options: serialize_when_none = False class User(NewUser): """Existing user.""" contact = ModelType(Contact) date_insert = DateTimeType(serialized_format=helpers.RFC3339Milli, tzd=u'utc') family_name = StringType() given_name = StringType() password = StringType() # not outpout by default, not required privacy_features = DictType(StringType, default=lambda: {}, ) pi = ModelType(PIParameter) user_id = UUIDType() class Options: roles = {'default': blacklist('password', 'settings')} serialize_when_none = False class NewRule(Model): """New filter rule.""" name = StringType(required=True) expr = StringType(required=True) position = IntType() stop_condition = BooleanType(default=False) tags = ListType(StringType) class Options: serialize_when_none = False ================================================ FILE: src/backend/main/py.main/caliopen_main/user/returns/__init__.py ================================================ # from .user import ReturnUser # # from .contact import ReturnContact, ReturnShortContact # from .contact import ReturnEmail, ReturnIM, ReturnPhone # from .contact import ReturnAddress, ReturnOrganization # from .contact import ReturnPublicKey, ReturnSocialIdentity __all__ = [ 'ReturnUser', 'ReturnContact', 'ReturnShortContact', 'ReturnEmail', 'ReturnIM', 'ReturnPhone', 'ReturnAddress', 'ReturnOrganization', 'ReturnPublicKey', 'ReturnSocialIdentity' ] ================================================ FILE: src/backend/main/py.main/caliopen_main/user/returns/user.py ================================================ # -*- coding: utf-8 -*- """User return object structure.""" from __future__ import absolute_import, print_function, unicode_literals from caliopen_storage.parameters import ReturnCoreObject from ..core import User, UserIdentity from ..parameters import User as UserParam from ..parameters import UserIdentity as UserIdentityParam class ReturnUser(ReturnCoreObject): """Return object for ``User`` core.""" _core_class = User _return_class = UserParam class ReturnUserIdentity(ReturnCoreObject): """Return object for ``UserIdentity`` core.""" _core_class = UserIdentity _return_class = UserIdentityParam ================================================ FILE: src/backend/main/py.main/caliopen_main/user/store/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .user import User, UserName, ReservedName, FilterRule, UserRecoveryEmail from .user import IndexUser, Settings from .identity import UserIdentity, IdentityLookup, IdentityTypeLookup from .tag import UserTag __all__ = [ 'User', 'UserName', 'UserRecoveryEmail', 'UserTag', 'FilterRule', 'ReservedName', 'UserIdentity', 'IdentityLookup', 'IdentityTypeLookup', 'IndexUser', 'UserTag', 'Settings', ] ================================================ FILE: src/backend/main/py.main/caliopen_main/user/store/identity.py ================================================ # -*- coding: utf-8 -*- """Caliopen cassandra models related to device.""" from __future__ import absolute_import, print_function, unicode_literals import logging from caliopen_storage.store.model import BaseModel from cassandra.cqlengine import columns log = logging.getLogger(__name__) class UserIdentity(BaseModel): """User's identities model.""" user_id = columns.UUID(primary_key=True) identity_id = columns.UUID(primary_key=True) credentials = columns.Map(columns.Text, columns.Text) display_name = columns.Text() identifier = columns.Text() infos = columns.Map(columns.Text, columns.Text) last_check = columns.DateTime() protocol = columns.Text() status = columns.Text() type = columns.Text() class IdentityLookup(BaseModel): """Model for identity_lookup table to retrieve identity by identifier""" identifier = columns.Text(primary_key=True) protocol = columns.Text(primary_key=True) user_id = columns.UUID(primary_key=True) identity_id = columns.UUID() class IdentityTypeLookup(BaseModel): """Model for identity_type_lookup table to retrieve identity by type""" type = columns.Text(primary_key=True) user_id = columns.UUID(primary_key=True) identity_id = columns.UUID(primary_key=True) ================================================ FILE: src/backend/main/py.main/caliopen_main/user/store/tag.py ================================================ # -*- coding: utf-8 -*- """Caliopen tag objects.""" from __future__ import absolute_import, print_function, unicode_literals from cassandra.cqlengine import columns from caliopen_storage.store import BaseModel class UserTag(BaseModel): """User tags model.""" user_id = columns.UUID(primary_key=True) name = columns.Text(primary_key=True) date_insert = columns.DateTime() importance_level = columns.Integer() label = columns.Text() type = columns.Text() ================================================ FILE: src/backend/main/py.main/caliopen_main/user/store/user.py ================================================ # -*- coding: utf-8 -*- """Caliopen cassandra objects related to user.""" from __future__ import absolute_import, print_function, unicode_literals import logging import uuid from cassandra.cqlengine import columns from caliopen_storage.store.model import BaseModel from caliopen_main.pi.objects import PIModel log = logging.getLogger(__name__) class UserName(BaseModel): """Maintain unicity of user name and permit lookup to user_id.""" name = columns.Text(primary_key=True) user_id = columns.UUID(required=True) class UserRecoveryEmail(BaseModel): """Permit user lookup by recovery_email.""" recovery_email = columns.Text(primary_key=True) user_id = columns.UUID(required=True) class ReservedName(BaseModel): """List of reserved user names.""" name = columns.Text(primary_key=True) class User(BaseModel): """User main model.""" user_id = columns.UUID(primary_key=True, default=uuid.uuid4) name = columns.Text(required=True) password = columns.Text(required=True) date_insert = columns.DateTime() date_delete = columns.DateTime() given_name = columns.Text() family_name = columns.Text() params = columns.Map(columns.Text, columns.Text) contact_id = columns.UUID() main_user_id = columns.UUID() recovery_email = columns.Text(required=True) shard_id = columns.Text() privacy_features = columns.Map(columns.Text(), columns.Text()) pi = columns.UserDefinedType(PIModel) class FilterRule(BaseModel): """User filter rules model.""" user_id = columns.UUID(primary_key=True) rule_id = columns.UUID(primary_key=True) date_insert = columns.DateTime() name = columns.Text() filter_expr = columns.Text() position = columns.Integer() stop_condition = columns.Boolean() class Settings(BaseModel): """All settings related to an user.""" user_id = columns.UUID(primary_key=True) default_locale = columns.Text() message_display_format = columns.Text() contact_display_format = columns.Text() contact_display_order = columns.Text() notification_enabled = columns.Boolean() notification_message_preview = columns.Text() notification_sound_enabled = columns.Boolean() notification_delay_disappear = columns.Integer() class IndexUser(object): """User index management class.""" ================================================ FILE: src/backend/main/py.main/caliopen_main/user/store/user_index.py ================================================ # -*- coding: utf-8 -*- """Caliopen index classes for nested tag.""" from __future__ import absolute_import, print_function, unicode_literals import logging from elasticsearch_dsl import InnerObjectWrapper, Keyword, Text from caliopen_storage.store.model import BaseIndexDocument log = logging.getLogger(__name__) class IndexedUser(BaseIndexDocument): doc_type = 'indexed_local_identity' display_name = Text() identifier = Text() status = Keyword() type = Keyword() class IndexedIdentity(InnerObjectWrapper): """nested identity within a message""" identifier = Text() type = Keyword() ================================================ FILE: src/backend/main/py.main/requirements.deps ================================================ caliopen_storage caliopen_pi ================================================ FILE: src/backend/main/py.main/setup.cfg ================================================ [nosetests] match = ^test nocapture = 1 cover-package = caliopen with-coverage = 1 cover-erase = 1 ================================================ FILE: src/backend/main/py.main/setup.py ================================================ import os import re from setuptools import setup, find_packages here = os.path.abspath(os.path.dirname(__file__)) README = open(os.path.join(here, 'README.rst')).read() CHANGES = open(os.path.join(here, 'CHANGES.rst')).read() name = "caliopen_main" with open(os.path.join(*([here] + name.split('.') + ['__init__.py']))) as v_file: version = re.compile(r".*__version__ = '(.*?)'", re.S).match(v_file.read()).group(1) requires = [ 'phonenumbers', 'pytz', 'zxcvbn_python', 'validate_email', 'uuid', 'regex', 'zope.interface', 'vobject', 'minio<5', ] if (os.path.isfile('./requirements.deps')): with open('./requirements.deps') as f_deps: requires.extend(f_deps.read().split('\n')) extras_require = { 'dev': [], 'test': [ 'coverage', 'docker-py', 'freezegun', 'nose' ], } setup(name=name, version=version, description='Caliopen main package. Entry point for whole application', long_description=README + '\n\n' + CHANGES, classifiers=["Programming Language :: Python", ], author='Caliopen contributors', author_email='contact@caliopen.org', url='https://caliopen.org', license='AGPLv3', packages=find_packages(), include_package_data=True, zip_safe=False, extras_require=extras_require, install_requires=requires, tests_require=requires, ) ================================================ FILE: src/backend/main/py.storage/CHANGES.rst ================================================ 0.0.1 ----- - Initial version ================================================ FILE: src/backend/main/py.storage/MANIFEST.in ================================================ include *.cfg *.rst *.template ================================================ FILE: src/backend/main/py.storage/README.rst ================================================ Entry point =========== This repository is part of CaliOpen platform. For documentation, installation and contribution instructions, please refer to https://caliopen.github.io Caliopen Storage ============= This is the base storage package for caliopen platform. It contains following sub packages: store All classes related to datastore. Base model User and Contact are included. core Classes where business logic must be define. Datastores objects are not directly managed, they must have a related core class to act as an interface with others caliopen components. helpers Some common helpers for all caliopen parts. Notes ----- waitress and cassandra driver conflict ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Cassandra python driver use async_core by default and can conflict with waitress event loop (1). It is advocated to install libev on your system to avoid this problem (2). (1) https://github.com/Pylons/waitress/issues/63 (2) http://datastax.github.io/python-driver/installation.html#c-extensions ================================================ FILE: src/backend/main/py.storage/caliopen_storage/__init__.py ================================================ # -*- coding: utf-8 -*- __version__ = '0.23.0' ================================================ FILE: src/backend/main/py.storage/caliopen_storage/config.py ================================================ # -*- coding: utf-8 -*- """Caliopen configuration class.""" from __future__ import unicode_literals, absolute_import import yaml try: from yaml import CSafeLoader as YAMLLoader except ImportError: from yaml import SafeLoader as YAMLLoader class Configuration(object): """ Configuration store.""" _conffiles = {} _filename = None def __init__(self, name): self._name = name @classmethod def load(cls, filename, name=None): """ Load configuration from `filename`. An optional `name` is recommended to use many environment. """ name = name or filename if name not in cls._conffiles: with open(filename) as fdesc: cls._conffiles[name] = yaml.load(fdesc, YAMLLoader) return cls(name) @property def configuration(self): """ Get the configuration for current object. .. deprecated:: use the :meth:`get` instead """ return self._conffiles[self._name] def get(self, key, default=None, separator='.'): """ Retrieve a configuration setting. :param key: a dot separated string :type key: str """ key = key.split(separator) value = self.configuration try: for k in key: value = value[k] return value except KeyError: return default ================================================ FILE: src/backend/main/py.storage/caliopen_storage/core/__init__.py ================================================ from .registry import core_registry from .base import BaseCore from .mixin import MixinCoreRelation __all__ = [ 'core_registry', 'BaseCore', 'MixinCoreRelation'] ================================================ FILE: src/backend/main/py.storage/caliopen_storage/core/base.py ================================================ # -*- coding: utf-8 -*- """ Caliop core base class. Core are glue code to the storage layer. """ from __future__ import absolute_import, print_function, unicode_literals from six import add_metaclass from uuid import UUID import logging from ..exception import NotFound from ..core.registry import core_registry log = logging.getLogger(__name__) class CoreMetaClass(type): """ Metaclass for all core. For all core classes related to a model, add it to core_registry. """ def __init__(cls, name, bases, namespace): super(CoreMetaClass, cls).__init__(name, bases, namespace) if cls._model_class: table_name = cls._model_class.__name__ if not core_registry.get(table_name): core_registry.update({table_name: cls}) @add_metaclass(CoreMetaClass) class BaseCore(object): """Base class for all core objects.""" _model_class = None _lookup_class = None _pkey_name = 'id' def __init__(self, model): """Initialize a core object with a model.""" self.model = model @classmethod def create(cls, **attrs): """Create a core object.""" obj = cls._model_class.create(**attrs) return cls(obj) @classmethod def get(cls, key): """Get a core object by key.""" params = {cls._pkey_name: key} obj = cls._model_class.get(**params) if obj: return cls(obj) raise NotFound('%s #%s not found' % (cls._model_class.__name__, key)) def save(self): """Save a core object.""" return self.model.save() def delete(self): """Delete a core object.""" # XXX delete related object (relation, lookup) return self.model.delete() def __getattr__(self, attr): """ used to proxy model attribute. Does not proxy attributed retrieve via a "lookup". """ if attr in self.model._columns.keys(): value = getattr(self.model, attr) if isinstance(value, UUID): return str(value) return value def get_id(self): """Return object id defined as its primary key.""" return getattr(self, self._pkey_name) @classmethod def find(cls, **kwargs): """Find core objects, can only use columns part of primary key.""" if 'count' in kwargs: count = kwargs.pop('count') else: count = False if not kwargs: objs = cls._model_class.all() else: objs = cls._model_class.filter(**kwargs) if count: return objs.count() return [cls(x) for x in objs] @classmethod def count(cls, **kwargs): """Count core objects matching filters.""" kwargs['count'] = True return cls.find(**kwargs) ================================================ FILE: src/backend/main/py.storage/caliopen_storage/core/mixin.py ================================================ # -*- coding: utf-8 -*- """Caliop core mixin classes.""" from __future__ import absolute_import, print_function, unicode_literals import logging import uuid from ..exception import NotFound log = logging.getLogger(__name__) class MixinCoreRelation(object): """Mixin to manage relations on core object.""" def _expand_relation(self, reltype): """Return collection for given relation.""" res = self._relations[reltype].find(self.user, self) return res['data'] if res else [] def _get_relation(self, reltype, id): """Get a specific core by in in relation.""" rel_pkey = self._relations[reltype]._pkey_name result = self._relations[reltype].find(self.user, self, {rel_pkey: id}) return result['data'][0] if result and result['data'] else None def _add_relation(self, reltype, param): """Add a new core to given relation.""" param.validate() if hasattr(param, 'is_primary') and param.is_primary: existing = self._expand_relation(reltype) for obj in existing: if obj.is_primary: obj.is_primary = False obj.save() # XXX don't forget to update index # Transform param into core object # XXX find a better method ? attrs = {k: v for k, v in param.to_primitive().iteritems() if v is not None} new_obj = self._relations[reltype].create(self.user, self, **attrs) rel_list = getattr(self, reltype) rel_list.append(new_obj.get_id()) self.save() if self._index_class: self._add_relation_index(reltype, attrs) if hasattr(self, '_lookup_objects') and \ reltype in self._lookup_objects: lookupkls = self._lookup_class look = lookupkls.create(user_id=self.user.user_id, value=new_obj.get_id(), contact_id=self.contact_id, type=reltype, lookup_id=new_obj.get_id()) log.debug('Created lookup object for relation %s:%r' % (reltype, look)) return new_obj def _delete_relation(self, reltype, id): """Delete core from relation.""" rel_list = getattr(self, reltype) if id in rel_list: rel_list.remove(id) self.save() related = self._get_relation(reltype, id) if self._index_class and related: pkey = related._pkey_name self._delete_relation_index(reltype, pkey, id) if related: related.model.delete() else: raise NotFound if hasattr(self, '_lookup_objects') and \ reltype in self._lookup_objects: lookupkls = self._lookup_class lookup = lookupkls.get(self.user, id) if lookup: lookup.delete() else: log.warn('Lookup object not found when deleting relation') return True def _add_relation_index(self, reltype, attrs): """Add a relation to indexed object.""" idx = self._index_class.get(self.user_id, self.get_id()) nested = getattr(idx, reltype) if not nested: log.warn('Nested index {} not found for {}'.format(reltype, self)) return False nested.append(attrs) return True def _delete_relation_index(self, reltype, key, id): """Delete a relation from an indexed object.""" idx = self._index_class.get(self.user_id, self.get_id()) # Look for existing entry found = None nested = getattr(idx, reltype) if not nested: log.warn('Nested index {} not found for {}'.format(reltype, self)) return False for child in nested: if getattr(child, key) == id: found = child if not found: log.warn('Relation %s %s with id %s not found in index' % (reltype, key, id)) nested.remove(found) return True class MixinCoreNested(object): """Mixin class for core nested objects management.""" def _add_nested(self, column, nested): """Add a nested object to a list.""" nested.validate() kls = self._nested.get(column) if not kls: raise Exception('No nested class for {}'.format(column)) column = getattr(self.model, column) # Ensure unicity if hasattr(kls, 'uniq_name'): for value in column: uniq = getattr(value, kls.uniq_name) if uniq == getattr(nested, kls.uniq_name): raise Exception('Unicity conflict for {}'.format(uniq)) if hasattr(nested, 'is_primary') and nested.is_primary: for old_primary in column: column.is_primary = False value = nested.to_primitive() pkey = getattr(kls, '_pkey') value[pkey] = uuid.uuid4() log.debug('Will insert nested {} : {}'.format(column, value)) column.append(kls(**value)) return value def _delete_nested(self, column, nested_id): """Delete a nested object with its id from a list.""" attr = getattr(self, column) log.debug('Will delete {} with id {}'.format(column, nested_id)) found = -1 for pos in xrange(0, len(attr)): nested = attr[pos] current_id = str(getattr(nested, nested._pkey)) if current_id == nested_id: found = pos if found == -1: log.warn('Nested object {}#{} not found for deletion'. format(column, nested_id)) return None return attr.pop(found) @classmethod def create_nested(cls, values, kls): """Create nested objects in store format.""" nested = [] for param in values: param.validate() attrs = param.to_primitive() if hasattr(kls, '_pkey'): # XXX default value not correctly handled attrs[getattr(kls, '_pkey')] = uuid.uuid4() nested.append(kls(**attrs)) return nested ================================================ FILE: src/backend/main/py.storage/caliopen_storage/core/registry.py ================================================ # -*- coding: utf-8 -*- """Caliop core registry.""" core_registry = {} # registry for all core classes ================================================ FILE: src/backend/main/py.storage/caliopen_storage/exception.py ================================================ # -*- coding: utf-8 -*- """Caliopen cassandra models related to user.""" from __future__ import absolute_import, print_function, unicode_literals class NotFound(Exception): """Exception when object is not found in store.""" pass class CredentialException(Exception): """Generic exception during user authentication process.""" pass class DuplicateObject(Exception): """Exception when an existing object already exists.""" pass ================================================ FILE: src/backend/main/py.storage/caliopen_storage/helpers/__init__.py ================================================ ================================================ FILE: src/backend/main/py.storage/caliopen_storage/helpers/connection.py ================================================ # -*- coding: utf-8 -*- """Caliopen storage session helpers.""" from cassandra.cqlengine.connection import setup as setup_cassandra from elasticsearch import Elasticsearch from ..config import Configuration def connect_storage(): """Connect to storage engines.""" try: from cassandra.io.libevreactor import LibevConnection kwargs = {'connection_class': LibevConnection} except ImportError: kwargs = {} hosts = Configuration('global').get('cassandra.hosts') keyspace = Configuration('global').get('cassandra.keyspace') consistency = Configuration('global').get('cassandra.consistency_level') protocol = Configuration('global').get('cassandra.protocol_version') setup_cassandra(hosts, keyspace, consistency, lazy_connect=True, protocol_version=protocol, **kwargs) def get_index_connection(): """Return a connection to index store.""" url = Configuration('global').get('elasticsearch.url') return Elasticsearch(url) ================================================ FILE: src/backend/main/py.storage/caliopen_storage/helpers/json.py ================================================ # -*- coding: utf-8 -*- """Caliopen helpers related to json data.""" # Strange situation under python 2.x where json do not have JSONENcoder try: import simplejson as json except ImportError: import json import datetime from uuid import UUID from decimal import Decimal class JSONEncoder(json.JSONEncoder): """Specific json encoder to deal with some specific types.""" _datetypes = (datetime.date, datetime.datetime) def default(self, obj): """Convert object to JSON encodable type.""" if isinstance(obj, Decimal): return float(obj) if isinstance(obj, self._datetypes): return RFC3339Milli(obj) if isinstance(obj, UUID): return str(obj) return super(JSONEncoder, self).default(obj) def to_json(data): """Helper to dump using correct encoder.""" return json.dumps(data, cls=JSONEncoder) def RFC3339Milli(value): return u"%s.%sZ" % ( value.strftime("%Y-%m-%dT%H:%M:%S"), value.microsecond / 1000) ================================================ FILE: src/backend/main/py.storage/caliopen_storage/parameters.py ================================================ # -*- coding: utf-8 -*- """Caliopen parameters classes. All input and ouput parameters of core object methods must use a class that inherit from of of these. """ import logging from .core import BaseCore log = logging.getLogger(__name__) class BaseReturnObject(object): """Base return object.""" _return_class = None _aliases = {} class ReturnCoreObject(BaseReturnObject): """Return object for core instance. to define a return object for a core suitable for protocol output, inherit from this class defining _core_class, _return_class attribute. parameter = ReturnObject.build(core) parameter.serialize() """ _core_class = None @classmethod def _build_sub_core(cls, core): """Build return for a core object attached to another.""" attr = {} for col in core._model_class._columns.keys(): value = getattr(core, col) if isinstance(value, (list, tuple)): new_value = [] for val in value: if hasattr(val, 'to_dict'): new_value.append(val.to_dict()) else: new_value.append(val) value = new_value elif hasattr(value, 'to_dict'): value = value.to_dict() attr.update({col: value}) return attr @classmethod def build(cls, core): """Main method to build a return object from a core.""" kls = cls._return_class obj = kls() for k, v in obj._data.iteritems(): if cls._aliases.get(k): core_key = cls._aliases[k] else: core_key = k attr = getattr(core, core_key) if hasattr(cls._core_class, '_relations') \ and k in cls._core_class._relations: # XXX bad design using data key if 'data' in attr: attr = attr['data'] attr = [x.to_dict() for x in attr] if isinstance(attr, BaseCore): attr = cls._build_sub_core(attr) elif isinstance(attr, (list, tuple)): new_attr = [] for val in attr: if hasattr(val, 'to_dict'): value = val.to_dict() else: value = val new_attr.append(value) attr = new_attr if attr is None and v is not None: setattr(obj, k, v) else: if hasattr(attr, 'to_dict'): setattr(obj, k, attr.to_dict()) else: setattr(obj, k, attr) obj.validate() return obj class ReturnIndexObject(BaseReturnObject): """Return object from index entry. Inherit from this class for an index class and define which parameter return class to use for build parameter = ReturnIndexObject(index_entry) parameter.serialize() """ _index_class = None _default = {} @classmethod def build(cls, entry): """Main method to build a return object from an index entry.""" kls = cls._return_class obj = kls() for k, v in obj._data.iteritems(): if cls._aliases.get(k): idx_key = cls._aliases[k] else: idx_key = k attr = entry.get(idx_key, cls._default.get(idx_key)) setattr(obj, k, attr) obj.validate() return obj ================================================ FILE: src/backend/main/py.storage/caliopen_storage/returns.py ================================================ ================================================ FILE: src/backend/main/py.storage/caliopen_storage/store/__init__.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals from .model import BaseModel, BaseIndexDocument, BaseUserType __all__ = [ 'BaseModel', 'BaseIndexDocument', 'BaseUserType', ] ================================================ FILE: src/backend/main/py.storage/caliopen_storage/store/mixin.py ================================================ """Caliopen mixins related to store.""" from __future__ import absolute_import, print_function, unicode_literals import logging import uuid from cassandra.cqlengine import columns log = logging.getLogger(__name__) def get_user_index(user_id): from caliopen_main.user.store import User log.warning('We should not be there to get user.shard_id') user = User.get(user_id=user_id) return user.shard_id class IndexedModelMixin(object): """Mixin to transform model into indexed documents.""" def __process_udt(self, column, idx): """Process a cassandra UDT column to translate into nested index.""" def map_udt_attributes(item): ret = {} for col_name, col_value in item.items(): if col_value is not None: if isinstance(col_value, (columns.UUID, uuid.UUID)): value = str(col_value) else: value = col_value ret[col_name] = value return ret attr_udt = getattr(self, column.column_name) if isinstance(attr_udt, list): udts = [] for item in attr_udt: udts.append(map_udt_attributes(item)) setattr(idx, column.column_name, udts) else: if attr_udt: setattr(idx, column.column_name, map_udt_attributes(attr_udt)) def _process_column(self, column, idx): """Process a core column and translate into index document.""" col_name = column.column_name col_value = getattr(self, col_name) try: getattr(idx, col_name) except AttributeError: log.debug('No such column in index mapping {}'. format(column.column_name)) return if isinstance(column, columns.List): is_udt = isinstance(column.sub_types[0], columns.UserDefinedType) if is_udt: self.__process_udt(column, idx) else: setattr(idx, col_name, col_value) elif isinstance(column, columns.UserDefinedType): self.__process_udt(column, idx) else: setattr(idx, col_name, col_value) def create_index(self, **extras): """Translate a model object into an indexed document.""" if not self._index_class: return False idx = self._index_class() # XXX TODO TEMPORARY FIX # Design on core object did not follow correctly user.shard_id logic idx.meta.index = get_user_index(self.user_id) for name, desc in self._columns.items(): if desc.is_primary_key: if name != 'user_id': idx.meta.id = getattr(self, name) else: self._process_column(desc, idx) else: self._process_column(desc, idx) for k, v in extras.items(): setattr(idx, k, v) idx.save(using=idx.client()) return True def update_index(self, object_id, changed_columns): """Update an existing index with a list of new values""" idx = self._index_class() idx.meta.index = get_user_index(self.user_id) idx.meta.id = object_id update_doc = {} for name in changed_columns: if name == 'user_id': raise Exception('Can not change user_id column') column = self._columns[name] self._process_column(column, idx) update_doc[name] = getattr(idx, name) # serialize index doc keeping empty or None value out = {} for k, v in idx._d_.iteritems(): try: f = idx._doc_type.mapping[k] if f._coerce: v = f.serialize(v) except KeyError: pass out[k] = v # XXX This method is no more used, deprecate it smoothly log.warning('Deprecation warning on IndexedModelMixin.update_index') idx.update(using=idx.client(), **out) @classmethod def search(cls, user, limit=None, offset=0, min_pi=0, max_pi=0, sort=None, **params): """Search in index using a dict parameter.""" search = cls._index_class.search(using=cls._index_class.client(), index=user.shard_id) search.filter('term', user_id=user.user_id) for k, v in params.items(): term = {k: v} search = search.filter('match', **term) if limit: search = search[offset:offset + limit] else: log.warn('Pagination not set for search query,' ' using default storage one') if sort: search = search.sort(sort) log.debug('Search is {}'.format(search.to_dict())) resp = search.execute() log.debug('Search result {}'.format(resp)) # XXX This method is no more used, deprecate it smoothly log.warning('Deprecation warning on IndexedModelMixin.update_index') return resp ================================================ FILE: src/backend/main/py.storage/caliopen_storage/store/model.py ================================================ # -*- coding: utf-8 -*- """Caliopen cassandra base model classes.""" from __future__ import absolute_import, print_function, unicode_literals import logging from cassandra.cqlengine.models import Model from cassandra.cqlengine.query import DoesNotExist from cassandra.cqlengine.usertype import UserType from elasticsearch import Elasticsearch from elasticsearch_dsl import DocType from ..config import Configuration from ..exception import NotFound log = logging.getLogger(__name__) class BaseModel(Model): """Cassandra base model.""" __abstract__ = True __keyspace__ = Configuration('global').get('cassandra.keyspace') _index_class = None @classmethod def create(cls, **kwargs): """Create a new model record.""" attrs = {key: val for key, val in kwargs.items() if key in cls._columns} obj = super(BaseModel, cls).create(**attrs) if obj._index_class: extras = kwargs.get('_indexed_extra', {}) obj.create_index(**extras) return obj @classmethod def get(cls, **kwargs): """Raise our exception when model not found.""" try: return super(BaseModel, cls).get(**kwargs) except DoesNotExist as exc: raise NotFound(exc) except: raise @classmethod def filter(cls, **kwargs): """Filter storable objects.""" return cls.objects.filter(**kwargs) @classmethod def all(cls): """Return all storable objects.""" return cls.objects.all() class BaseIndexDocument(DocType): """Base class for indexed objects.""" doc_type = None __url__ = Configuration('global').get('elasticsearch.url') @classmethod def client(cls): """Return an elasticsearch client.""" return Elasticsearch(cls.__url__) @classmethod def create_mapping(cls, index_name): """Create and save elasticsearch mapping for the cls.doc_type.""" if hasattr(cls, 'build_mapping'): log.info('Create index {} mapping for doc_type {}'. format(index_name, cls.doc_type)) try: cls.build_mapping().save(using=cls.client(), index=index_name) except Exception as exc: log.error("failed to put mapping for {} : {}". format(index_name, exc)) raise exc class BaseUserType(UserType): """Base class for UserDefined Type in store layer.""" def to_dict(self): return {k: v for k, v in self.items()} ================================================ FILE: src/backend/main/py.storage/setup.cfg ================================================ [nosetests] match = ^test nocapture = 1 cover-package = caliopen with-coverage = 1 cover-erase = 1 ================================================ FILE: src/backend/main/py.storage/setup.py ================================================ import os import re from setuptools import setup, find_packages here = os.path.abspath(os.path.dirname(__file__)) README = open(os.path.join(here, 'README.rst')).read() CHANGES = open(os.path.join(here, 'CHANGES.rst')).read() name = "caliopen_storage" with open(os.path.join(*([here] + name.split('.') + ['__init__.py']))) as v_file: version = re.compile(r".*__version__ = '(.*?)'", re.S).match(v_file.read()).group(1) requires = [ 'setuptools', 'cryptography==2.4', 'six == 1.10.0', # https://github.com/SecurityInnovation/PGPy/issues/217 'bcrypt', 'PyYAML', 'elasticsearch-dsl>=5.0.0,<6.0.0', 'cassandra-driver==3.4.1', 'schematics', 'simplejson', 'jsonschema == 2.6.0', ] extras_require = { 'dev': [], 'test': ['nose', 'coverage', 'freezegun', 'docker-py'], } setup(name=name, version=version, description='Caliopen base package for storage routines.', long_description=README + '\n\n' + CHANGES, classifiers=["Programming Language :: Python", ], author='Caliopen contributors', author_email='contact@caliopen.org', url='https://caliopen.org', license='AGPLv3', packages=find_packages(), include_package_data=True, zip_safe=False, extras_require=extras_require, install_requires=requires, tests_require=requires, ) ================================================ FILE: src/backend/protocols/go.imap/cmd/imapctl/cli_cmds/addremote.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package cmd import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/cassandra" log "github.com/Sirupsen/logrus" "github.com/gocql/gocql" "github.com/satori/go.uuid" "github.com/spf13/cobra" ) var ( id remoteId addRemoteCmd = &cobra.Command{ Use: "addremote", Short: "add an IMAP remote identity for specified user", Run: addRemote, } ) type remoteId struct { DisplayName string Login string Mailbox string Password string PollInterval string RemoteId string Server string UserId UUID UserName string } func init() { //mandatory addRemoteCmd.Flags().StringVarP(&id.UserName, "username", "u", "", "user name account in which mails will be imported (required)") addRemoteCmd.Flags().StringVarP(&id.Server, "server", "s", "", "remote hostname[:port] IMAP server address (required)") addRemoteCmd.Flags().StringVarP(&id.Login, "login", "l", "", "IMAP login credential (required)") //optional addRemoteCmd.Flags().StringVarP(&id.Password, "pass", "p", "", "IMAP password credential") addRemoteCmd.Flags().StringVarP(&id.Mailbox, "mailbox", "m", "INBOX", "IMAP mailbox name to fetch mail from (case sensitive, default to 'INBOX'") addRemoteCmd.Flags().StringVarP(&id.DisplayName, "display", "d", "", "display name for remote identity (default to login)") addRemoteCmd.MarkFlagRequired("username") addRemoteCmd.MarkFlagRequired("server") addRemoteCmd.MarkFlagRequired("login") RootCmd.AddCommand(addRemoteCmd) } func addRemote(cmd *cobra.Command, args []string) { var is backends.IdentityStorage var us backends.UserNameStorage var cb *store.CassandraBackend var rId UserIdentity var err error switch cmdConfig.StoreName { case "cassandra": c := store.CassandraConfig{ Hosts: cmdConfig.StoreConfig.Hosts, Keyspace: cmdConfig.StoreConfig.Keyspace, Consistency: gocql.Consistency(cmdConfig.StoreConfig.Consistency), SizeLimit: cmdConfig.StoreConfig.SizeLimit, } cb, err = store.InitializeCassandraBackend(c) if err != nil { log.WithError(err).Fatalf("[addRemote] initalization of %s backend failed", cmdConfig.StoreName) } } is = backends.IdentityStorage(cb) us = backends.UserNameStorage(cb) user, e := us.UserByUsername(id.UserName) if e != nil { log.WithError(e).Fatalf("[addRemote] failed to retrieve user name <%s>", id.UserName) } id.UserId = user.UserId if id.DisplayName == "" { id.DisplayName = id.Login } rId = UserIdentity{ DisplayName: id.DisplayName, Identifier: id.Login, Status: "active", Type: RemoteIdentity, Protocol: EmailProtocol, UserId: id.UserId, } rId.Id.UnmarshalBinary(uuid.NewV4().Bytes()) rId.SetDefaults() rId.Infos["inserver"] = id.Server rId.Credentials = &Credentials{ "inpassword": id.Password, "inusername": id.Login, } err = is.CreateUserIdentity(&rId) if err != nil { log.WithError(err).Warn("[addRemote] storage failed to store remote identity") } else { log.Infof("OK, new remote identity added with id %s ! Bye.", rId.Id.String()) } } ================================================ FILE: src/backend/protocols/go.imap/cmd/imapctl/cli_cmds/fullfetch.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package cmd import ( "encoding/json" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/cassandra" "github.com/Sirupsen/logrus" log "github.com/Sirupsen/logrus" "github.com/gocql/gocql" "github.com/nats-io/go-nats" "github.com/spf13/cobra" ) var ( fullFetchCmd = &cobra.Command{ Use: "fullfetch", Short: "blindly imports all mails from a remote IMAP mailbox into user's account.", Run: fullFetch, } ) func init() { fullFetchCmd.Flags().StringVarP(&id.UserName, "username", "u", "", "user_name account in which mails will be imported (required)") fullFetchCmd.Flags().StringVarP(&id.Server, "server", "s", "", "remote hostname[:port] IMAP server address (required)") fullFetchCmd.Flags().StringVarP(&id.Mailbox, "mailbox", "m", "INBOX", "IMAP mailbox name to fetch mail from, case sensitive") fullFetchCmd.Flags().StringVarP(&id.Login, "login", "l", "", "IMAP login credential (required)") fullFetchCmd.Flags().StringVarP(&id.Password, "pass", "p", "", "IMAP password credential (required)") fullFetchCmd.MarkFlagRequired("username") fullFetchCmd.MarkFlagRequired("server") fullFetchCmd.MarkFlagRequired("login") fullFetchCmd.MarkFlagRequired("pass") RootCmd.AddCommand(fullFetchCmd) } // fullFetch func fullFetch(cmd *cobra.Command, args []string) { var us backends.UserNameStorage var cb *store.CassandraBackend var err error switch cmdConfig.StoreName { case "cassandra": c := store.CassandraConfig{ Hosts: cmdConfig.StoreConfig.Hosts, Keyspace: cmdConfig.StoreConfig.Keyspace, Consistency: gocql.Consistency(cmdConfig.StoreConfig.Consistency), SizeLimit: cmdConfig.StoreConfig.SizeLimit, } cb, err = store.InitializeCassandraBackend(c) if err != nil { log.WithError(err).Fatalf("[addRemote] initalization of %s backend failed", cmdConfig.StoreName) } } us = backends.UserNameStorage(cb) user, e := us.UserByUsername(id.UserName) if e != nil { log.WithError(e).Fatalf("[addRemote] failed to retrieve user name <%s>", id.UserName) } id.UserId = user.UserId nc, err := nats.Connect(cmdConfig.NatsUrl) if err != nil { logrus.WithError(err).Fatal("nats connect failed") } defer nc.Close() msg, err := json.Marshal(IMAPorder{ Order: "fullfetch", UserId: id.UserId.String(), Server: id.Server, Mailbox: id.Mailbox, Login: id.Login, Password: id.Password, }) if err != nil { logrus.WithError(err).Fatal("unable to marshal natsOrder") } nc.Publish(cmdConfig.NatsTopicSender, msg) nc.Flush() if err := nc.LastError(); err != nil { logrus.WithError(err).Fatal("nats publish failed") } logrus.Infof("ordering to fetch all mails from %s for user %s", id.Login, id.UserId) } ================================================ FILE: src/backend/protocols/go.imap/cmd/imapctl/cli_cmds/root.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package cmd import ( imapWorker "github.com/CaliOpen/Caliopen/src/backend/protocols/go.imap" log "github.com/Sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" ) var ( cmdConfig CmdConfig configFile string configPath string verbose bool version bool RootCmd = &cobra.Command{ Use: "imapctl", Short: "cli for IMAP operations", Long: "IMAPctl is a cli to control IMAP related operations.", Run: nil, } ) const __version__ = "0.1.0" type CmdConfig imapWorker.WorkerConfig func init() { cobra.OnInitialize(initConfig) RootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "print out more debug information") RootCmd.PersistentFlags().BoolVarP(&version, "version", "V", false, "print out the version of this program") RootCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "imapworker", "Name of the configuration file, without extension. (YAML, TOML, JSON… allowed)") RootCmd.PersistentFlags().StringVarP(&configPath, "configpath", "", "../../../../configs/", "Main config file path.") RootCmd.Run = func(cmd *cobra.Command, args []string) { if version { log.Infof("IMAPctl version %s", __version__) } if len(args) == 0 { cmd.Help() } } RootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { if verbose { log.SetLevel(log.DebugLevel) } else { log.SetLevel(log.InfoLevel) } } RootCmd.AddCommand(versionCmd) } var versionCmd = &cobra.Command{ Use: "version", Short: "Print the version number of IMAPctl", Long: `All software has versions. This is IMAPctl's`, Run: func(cmd *cobra.Command, args []string) { log.Infof("IMAPctl version %s", __version__) }, } func initConfig() { // load in the main config. Reading from YAML, TOML, JSON, HCL and Java properties config files v := viper.New() v.SetConfigName(configFile) // name of config file (without extension) v.AddConfigPath(configPath) // path to look for the config file in v.AddConfigPath("$CALIOPENROOT/src/backend/configs/") // call multiple times to add many search paths v.AddConfigPath(".") // optionally look for config in the working directory err := v.ReadInConfig() // Find and read the config file*/ if err != nil { log.WithError(err).Fatalf("Could not read main config file <%s>.", configFile) } err = v.Unmarshal(&cmdConfig) if err != nil { log.WithError(err).Fatalf("Could not parse config file: <%s>", configFile) } cmdConfig.LDAConfig.AppVersion = __version__ } ================================================ FILE: src/backend/protocols/go.imap/cmd/imapctl/cli_cmds/syncremote.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package cmd import ( "encoding/json" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/cassandra" "github.com/Sirupsen/logrus" log "github.com/Sirupsen/logrus" "github.com/gocql/gocql" "github.com/nats-io/go-nats" "github.com/spf13/cobra" ) var ( syncRemoteCmd = &cobra.Command{ Use: "syncremote", Short: "sync remote mailbox for provided remote identity", Run: syncRemote, } ) func init() { syncRemoteCmd.Flags().StringVarP(&id.UserName, "username", "u", "", "Caliopen username account to which mails will be delivered (required)") syncRemoteCmd.Flags().StringVarP(&id.RemoteId, "remoteid", "r", "", "remote identity's uuid (required)") syncRemoteCmd.Flags().StringVarP(&id.Password, "pass", "p", "", "IMAP password (if not stored in db)") syncRemoteCmd.MarkFlagRequired("userid") syncRemoteCmd.MarkFlagRequired("remoteid") RootCmd.AddCommand(syncRemoteCmd) } func syncRemote(cmd *cobra.Command, args []string) { var us backends.UserNameStorage var cb *store.CassandraBackend var err error switch cmdConfig.StoreName { case "cassandra": c := store.CassandraConfig{ Hosts: cmdConfig.StoreConfig.Hosts, Keyspace: cmdConfig.StoreConfig.Keyspace, Consistency: gocql.Consistency(cmdConfig.StoreConfig.Consistency), SizeLimit: cmdConfig.StoreConfig.SizeLimit, } cb, err = store.InitializeCassandraBackend(c) if err != nil { log.WithError(err).Fatalf("[addRemote] initalization of %s backend failed", cmdConfig.StoreName) } } us = backends.UserNameStorage(cb) user, e := us.UserByUsername(id.UserName) if e != nil { log.WithError(e).Fatalf("[addRemote] failed to retrieve user name <%s>", id.UserName) } nc, err := nats.Connect(cmdConfig.NatsUrl) if err != nil { logrus.WithError(err).Fatal("nats connect failed") } defer nc.Close() msg, err := json.Marshal(IMAPorder{ Login: id.Login, Mailbox: id.Mailbox, Order: "sync", Password: id.Password, IdentityId: id.RemoteId, Server: id.Server, UserId: user.UserId.String(), }) if err != nil { logrus.WithError(err).Fatal("unable to marshal natsOrder") } nc.Publish(cmdConfig.NatsTopicSender, msg) nc.Flush() if err := nc.LastError(); err != nil { logrus.WithError(err).Fatal("nats publish failed") } logrus.Infof("ordering to sync mailbox from %s for user %s", id.DisplayName, user.UserId.String()) } ================================================ FILE: src/backend/protocols/go.imap/cmd/imapctl/main.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package main import ( "fmt" "github.com/CaliOpen/Caliopen/src/backend/protocols/go.imap/cmd/imapctl/cli_cmds" "os" ) func main() { if err := cmd.RootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(-1) } } ================================================ FILE: src/backend/protocols/go.imap/cmd/imapworker/cli_cmds/root.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package cmd import ( log "github.com/Sirupsen/logrus" "github.com/spf13/cobra" ) var ( verbose bool version bool RootCmd = &cobra.Command{ Use: "imapworker", Short: "IMAP worker daemon", Long: `IMAP worker is a daemon that subscribes to relevant messaging system queues and executes operations on remote IMAP servers`, Run: nil, } ) const __version__ = "0.23.0" func init() { cobra.OnInitialize() RootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "print out more debug information") RootCmd.PersistentFlags().BoolVarP(&version, "version", "V", false, "print out the version of this program") RootCmd.Run = func(cmd *cobra.Command, args []string) { if version { log.Infof("IMAP worker version %s", __version__) } if len(args) == 0 { cmd.Help() } } RootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { if verbose { log.SetLevel(log.DebugLevel) } else { log.SetLevel(log.InfoLevel) } } RootCmd.AddCommand(versionCmd) } var versionCmd = &cobra.Command{ Use: "version", Short: "Print the version number of IMAP worker", Long: `All software has versions. This is IMAP worker's`, Run: func(cmd *cobra.Command, args []string) { log.Infof("IMAP worker version %s", __version__) }, } ================================================ FILE: src/backend/protocols/go.imap/cmd/imapworker/cli_cmds/start.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package cmd import ( "crypto/rand" "fmt" imapWorker "github.com/CaliOpen/Caliopen/src/backend/protocols/go.imap" log "github.com/Sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" "io" "os" "os/signal" "sync" "syscall" "time" ) var ( configPath string configFile string pidFile string signalChannel chan os.Signal // for trapping SIG_HUP cmdConfig CmdConfig imapWorkers []*imapWorker.Worker startCmd = &cobra.Command{ Use: "start", Short: "Starts IMAP worker(s)", Run: start, } ) const ( shutdownTimeout = 3 // minutes to wait before forcing shutdown ) func init() { startCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "imapworker", "Name of the configuration file, without extension. (YAML, TOML, JSON… allowed)") startCmd.PersistentFlags().StringVarP(&configPath, "configpath", "", "../../../../configs/", "Main config file path.") startCmd.PersistentFlags().StringVarP(&pidFile, "pid-file", "p", "/var/run/caliopen_imap_worker.pid", "Path to the pid file") RootCmd.AddCommand(startCmd) signalChannel = make(chan os.Signal, 1) cmdConfig = CmdConfig{} } func sigHandler(workers []*imapWorker.Worker) { // handle SIGHUP for reloading the configuration while running signal.Notify(signalChannel, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGINT, syscall.SIGKILL) for sig := range signalChannel { if sig == syscall.SIGHUP { err := readConfig(&cmdConfig) if err != nil { log.WithError(err).Error("Error while ReadConfig (reload)") } else { log.Info("Configuration is reloaded") } // TODO: reinitialize } else if sig == syscall.SIGTERM || sig == syscall.SIGQUIT || sig == syscall.SIGINT || sig == syscall.SIGKILL { log.Infof("Shutdown signal caught. Gracefully halting %d workers within 3 minutes timeframe…", len(workers)) wg := new(sync.WaitGroup) wg.Add(len(workers)) for i := range workers { workers[i].HaltGroup = wg } // timeout mechanism to avoid infinite wait c := make(chan struct{}) go func() { defer close(c) wg.Wait() }() select { case <-c: log.Info("Shutdown completed, exiting") os.Exit(0) case <-time.After(shutdownTimeout * time.Minute): log.Warn("Shutdown timeout, force exiting") os.Exit(0) } } else { os.Exit(0) } } } func start(cmd *cobra.Command, args []string) { err := readConfig(&cmdConfig) if err != nil { log.WithError(err).Fatal("Error while reading config") } // Write out our PID if len(pidFile) > 0 { if f, err := os.Create(pidFile); err == nil { defer f.Close() if _, err := f.WriteString(fmt.Sprintf("%d", os.Getpid())); err == nil { f.Sync() } else { log.WithError(err).Warnf("Error while writing pidFile (%s)", pidFile) } } else { log.WithError(err).Warnf("Error while creating pidFile (%s)", pidFile) } } // init and start worker(s) var i uint8 imapWorkers = make([]*imapWorker.Worker, cmdConfig.Workers) for i = 0; i < cmdConfig.Workers; i++ { log.Infof("initializing IMAP worker %d", i) imapWorkers[i], err = imapWorker.NewWorker(imapWorker.WorkerConfig(cmdConfig), randomIdentifier()) if err != nil { log.WithError(err).Fatal("Failed to init IMAP Worker") } go imapWorkers[i].Start() } sigHandler(imapWorkers) } type CmdConfig imapWorker.WorkerConfig // ReadConfig which should be called at startup, or when a SIG_HUP is caught func readConfig(config *CmdConfig) error { // load in the main config. Reading from YAML, TOML, JSON, HCL and Java properties config files v := viper.New() v.SetConfigName(configFile) // name of config file (without extension) v.AddConfigPath(configPath) // path to look for the config file in v.AddConfigPath("$CALIOPENROOT/src/backend/configs/") // call multiple times to add many search paths v.AddConfigPath(".") // optionally look for config in the working directory err := v.ReadInConfig() // Find and read the config file*/ if err != nil { log.WithError(err).Infof("Could not read main config file <%s>.", configFile) return err } err = v.Unmarshal(config) if err != nil { log.WithError(err).Infof("Could not parse config file: <%s>", configFile) return err } config.LDAConfig.AppVersion = __version__ return nil } func randomIdentifier() string { var buf [4]byte _, err := io.ReadFull(rand.Reader, buf[:]) if err != nil { return "00000000" } return fmt.Sprintf("%x", buf[:]) } ================================================ FILE: src/backend/protocols/go.imap/cmd/imapworker/main.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package main import ( "fmt" "github.com/CaliOpen/Caliopen/src/backend/protocols/go.imap/cmd/imapworker/cli_cmds" "os" ) func main() { if err := cmd.RootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(-1) } } ================================================ FILE: src/backend/protocols/go.imap/config.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package imap_worker import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "time" ) type ( WorkerConfig struct { Hostname string `mapstructure:"hostname"` NatsQueue string `mapstructure:"nats_queue"` NatsTopicPoller string `mapstructure:"nats_topic_poller"` NatsTopicPollerCache string `mapstructure:"nats_topic_poller_cache"` NatsTopicSender string `mapstructure:"nats_topic_sender"` NatsUrl string `mapstructure:"nats_url"` StoreName string `mapstructure:"store_name"` Workers uint8 `mapstructure:"workers"` StoreConfig StoreConfig `mapstructure:"store_settings"` LDAConfig LDAConfig `mapstructure:"LDAConfig"` } ) const ( syncingTimeout = 24 // how many hours to wait before restarting sync op failuresThreshold = 72 // how many hours to wait before disabling a faulty remote pollThrottling = 10 * time.Second // how long to pause before requesting jobs again to idpoller ) ================================================ FILE: src/backend/protocols/go.imap/fetcher.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package imap_worker import ( "encoding/json" "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/Notifications" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/users" log "github.com/Sirupsen/logrus" "github.com/emersion/go-imap" "github.com/satori/go.uuid" "strconv" "time" ) type Fetcher struct { Hostname string Store backends.LDAStore Lda *Lda } type imapBox struct { lastSeenUid uint32 lastSync time.Time name string uidValidity uint32 } const ( lastErrorKey = "lastFetchError" dateFirstErrorKey = "firstErrorDate" dateLastErrorKey = "lastErrorDate" errorsCountKey = "errorsCount" ) // unexported vars to help override funcs in tests var syncRemoteWithLocal = func(f *Fetcher, order IMAPorder) error { return f.SyncRemoteWithLocal(order) } var fetchRemoteToLocal = func(f *Fetcher, order IMAPorder) error { return f.FetchRemoteToLocal(order) } // FetchSyncRemote retrieves remote identity credentials and last sync data, // connects to remote IMAP server to fetch new mails, // adds X-Fetched-Imap headers before forwarding mails to lda, // updates last sync data for identity in db. func (f *Fetcher) SyncRemoteWithLocal(order IMAPorder) error { log.Infof("[Fetcher] will fetch mails for remote %s", order.IdentityId) // 1. retrieve infos from db userIdentity, err := f.Store.RetrieveUserIdentity(order.UserId, order.IdentityId, true) if err != nil { log.WithError(err).Infof("[SyncRemoteWithLocal] failed to retrieve remote identity <%s> : <%s>", order.UserId, order.IdentityId) return err } if order.Password != "" { (*userIdentity.Credentials)["inpassword"] = order.Password } // 1.2 check if a sync process is running if syncing, ok := userIdentity.Infos["syncing"]; ok && syncing != "" { startDate, e := time.Parse(time.RFC3339, syncing) if e == nil && time.Since(startDate)/time.Hour < syncingTimeout { log.Infof("[SyncRemoteWithLocal] avoiding concurrent sync for <%s>. Syncing in progress since %s", order.IdentityId, userIdentity.Infos["syncing"]) return nil } } // save syncing state in db to prevent concurrent sync (*userIdentity).Infos["syncing"] = time.Now().Format(time.RFC3339) err = f.Store.UpdateUserIdentity(userIdentity, map[string]interface{}{ "Infos": userIdentity.Infos, }) if err != nil { log.WithError(err).Infof("[SyncRemoteWithLocal] failed to update remote identity <%s> : <%s>", order.UserId, order.IdentityId) return err } // 2. sync/fetch with remote IMAP mails := make(chan *Email) lastsync := time.Time{} if ls, ok := userIdentity.Infos["lastsync"]; ok && ls != "" { lastsync, err = time.Parse(time.RFC3339, userIdentity.Infos["lastsync"]) if err != nil { log.WithError(err).Warnf("[syncMails] failed to parse lastsync string <%s>", userIdentity.Infos["lastsync"]) lastsync = time.Time{} } } else { lastsync = time.Time{} } // Sync INBOX (only INBOX for now) // TODO : sync other mailbox(es) from userIdentity.Infos params or from order lastseenuid, _ := strconv.Atoi(userIdentity.Infos["lastseenuid"]) uidvalidity, _ := strconv.Atoi(userIdentity.Infos["uidvalidity"]) box := imapBox{ lastSeenUid: uint32(lastseenuid), lastSync: lastsync, name: "INBOX", uidValidity: uint32(uidvalidity), } go f.syncMails(userIdentity, &box, mails) // 3. forward mails to lda as they come on mails chan errs := []error{} syncTimeout := time.Now() batch := Notifications.NewBatch("imap_worker") for mail := range mails { if mail.ImapUid <= box.lastSeenUid { // do not forward seen message, we already have it continue } err := f.Lda.deliverMail(mail, order.UserId, userIdentity.Id.String(), batch) errs = append(errs, err) if err == nil { box.lastSeenUid = mail.ImapUid } if time.Since(syncTimeout)/time.Hour > syncingTimeout { errs = append(errs, errors.New("[Fetcher] sync timeout, aborting for "+order.IdentityId)) close(mails) break } } for i, err := range errs { // TODO: improve error handling protocol if err != nil { log.WithError(err).Warnf("[Fetcher] SyncRemoteWithLocal error delivering mail #%d", i) } } // 4. backup sync state in db var fields map[string]interface{} delete((*userIdentity).Infos, "syncing") userIdentity.LastCheck = time.Now() if _, ok := userIdentity.Infos[errorsCountKey]; !ok { // if errorsCountKey IS NOT in Infos then sync succeeded // update infos accordingly userIdentity.Infos["uidvalidity"] = strconv.Itoa(int(box.uidValidity)) userIdentity.Infos["lastsync"] = userIdentity.LastCheck.Format(time.RFC3339) userIdentity.Infos["lastseenuid"] = strconv.Itoa(int(box.lastSeenUid)) } fields = map[string]interface{}{ "LastCheck": userIdentity.LastCheck, "Infos": userIdentity.Infos, } err = f.Store.UpdateUserIdentity(userIdentity, fields) if err != nil { log.WithError(err).Warnf("[syncMails] failed to backup sync state") return err } log.Infof("[Fetcher] all done for %s : %d new mail(s) fetched", order.IdentityId, len(errs)) batch.Save(f.Lda.broker.Notifier, "", LongLived) return nil } // FetchRemoteToLocal blindly fetches all mails from remote without retrieving/saving any state in UserIdentity func (f *Fetcher) FetchRemoteToLocal(order IMAPorder) error { userIdentity := UserIdentity{ UserId: UUID(uuid.FromStringOrNil(order.UserId)), Infos: map[string]string{ "inserver": order.Server, }, Credentials: &Credentials{ "inusername": order.Login, "inpassword": order.Password, }, } box := imapBox{ lastSync: time.Time{}, name: order.Mailbox, } // 2. fetch remote messages mails := make(chan *Email, 10) go f.fetchMails(&userIdentity, &box, mails) //TODO errors handling // 3. forward mails to lda errs := make([]error, len(mails)) batch := Notifications.NewBatch("imap_worker") for mail := range mails { err := f.Lda.deliverMail(mail, order.UserId, order.IdentityId, batch) errs = append(errs, err) } return nil } // fetchMails fetches all messages from remote mailbox and returns well-formed Emails for lda. func (f *Fetcher) fetchMails(userIdentity *UserIdentity, box *imapBox, ch chan *Email) (err error) { if userIdentity.Infos["authtype"] == Oauth2 { err = users.ValidateOauth2Credentials(userIdentity, f, true) if err != nil { return f.handleFetchFailure(userIdentity, WrapCaliopenErr(err, WrongCredentialsErr, "Oauth2 validation failure")) } } tlsConn, imapClient, provider, err := imapLogin(userIdentity) // Don't forget to logout and close chan defer func() { imapClient.Logout() log.Println("Logged out") close(ch) }() if err != nil { return f.handleFetchFailure(userIdentity, WrapCaliopenErr(err, WrongCredentialsErr, "imapLogin failure")) } else { delete((*userIdentity).Infos, lastErrorKey) delete((*userIdentity).Infos, errorsCountKey) delete((*userIdentity).Infos, dateFirstErrorKey) delete((*userIdentity).Infos, dateLastErrorKey) } newMessages := make(chan *imap.Message, 10) go fetchMailbox(box, imapClient, provider, newMessages) //TODO : errors handling for msg := range newMessages { mail, err := MarshalImap(msg, buildXheaders(tlsConn, userIdentity, box, msg, provider)) if err != nil { //todo continue } ch <- mail } return } // fetchSyncMails reads last sync state for remote identity, // fetches new messages accordingly, and returns well-formed Emails for lda. func (f *Fetcher) syncMails(userIdentity *UserIdentity, box *imapBox, ch chan *Email) (err error) { // Don't forget to close chan before leaving defer close(ch) if userIdentity.Infos["authtype"] == Oauth2 { err = users.ValidateOauth2Credentials(userIdentity, f, true) if err != nil { return f.handleFetchFailure(userIdentity, WrapCaliopenErr(err, WrongCredentialsErr, "Oauth2 validation failure")) } } tlsConn, imapClient, provider, err := imapLogin(userIdentity) if err != nil { return f.handleFetchFailure(userIdentity, WrapCaliopenErr(err, WrongCredentialsErr, "imapLogin failure")) } else { delete((*userIdentity).Infos, lastErrorKey) delete((*userIdentity).Infos, errorsCountKey) delete((*userIdentity).Infos, dateFirstErrorKey) delete((*userIdentity).Infos, dateLastErrorKey) } // Don't forget to logout defer func() { imapClient.Logout() log.Println("Logged out") }() newMessages := make(chan *imap.Message, 10) //TODO : manage closing go syncMailbox(box, imapClient, provider, newMessages) //TODO : errors handling // read new messages coming from imap chan and write to lda chan with added custom headers for msg := range newMessages { xHeaders := buildXheaders(tlsConn, userIdentity, box, msg, provider) mail, err := MarshalImap(msg, xHeaders) if err != nil { //todo continue } else { ch <- mail } } return } // handleFetchFailure logs a warn and save failure log in db. // If failures reach failuresThreshold, remote id is disabled and a new notification is emitted. func (f *Fetcher) handleFetchFailure(userIdentity *UserIdentity, err CaliopenError) error { // ensure errors data fields are present if _, ok := userIdentity.Infos[lastErrorKey]; !ok { (*userIdentity).Infos[lastErrorKey] = "" } if _, ok := userIdentity.Infos[dateFirstErrorKey]; !ok { (*userIdentity).Infos[dateFirstErrorKey] = "" } if _, ok := userIdentity.Infos[dateLastErrorKey]; !ok { (*userIdentity).Infos[dateLastErrorKey] = "" } if _, ok := userIdentity.Infos[errorsCountKey]; !ok { (*userIdentity).Infos[errorsCountKey] = "0" } // log last error (*userIdentity).Infos[lastErrorKey] = "imap connection failed : " + err.Cause().Error() log.WithError(err.Cause()).Warnf("imap connection failed for remote identity %s", userIdentity.Id) // increment counter count, _ := strconv.Atoi(userIdentity.Infos[errorsCountKey]) count++ (*userIdentity).Infos[errorsCountKey] = strconv.Itoa(count) // update dates lastDate := time.Now() var firstDate time.Time firstDate, _ = time.Parse(time.RFC3339, userIdentity.Infos[dateFirstErrorKey]) if firstDate.IsZero() { firstDate = lastDate } (*userIdentity).Infos[dateFirstErrorKey] = firstDate.Format(time.RFC3339) (*userIdentity).Infos[dateLastErrorKey] = lastDate.Format(time.RFC3339) // check failuresThreshold if lastDate.Sub(firstDate)/time.Hour > failuresThreshold { f.disableRemoteIdentity(userIdentity) } // unlock sync state delete((*userIdentity).Infos, "syncing") // udpate UserIdentity in db return f.Store.UpdateUserIdentity(userIdentity, map[string]interface{}{ "Infos": userIdentity.Infos, "LastCheck": lastDate, }) } func (f *Fetcher) disableRemoteIdentity(userIdentity *UserIdentity) { (*userIdentity).Status = "inactive" err := f.Store.UpdateUserIdentity(userIdentity, map[string]interface{}{ "Status": "inactive", }) if err != nil { log.WithError(err).Warnf("[disableRemoteIdentity] failed to deactivate remote identity %s for user %s", userIdentity.Id, userIdentity.Id) } // send nats message to idpoller to stop polling order := RemoteIDNatsMessage{ IdentityId: userIdentity.Id.String(), Order: "delete", Protocol: "email", UserId: userIdentity.UserId.String(), } jorder, jerr := json.Marshal(order) if jerr == nil { e := f.Lda.broker.NatsConn.Publish(f.Lda.Config.NatsTopicPollerCache, jorder) if e != nil { log.WithError(e).Warnf("[saveErrorState] failed to publish delete order to idpoller") } } } func (f Fetcher) emitNotification() { //TODO } /* Oauth2Interfacer implementation */ func (f *Fetcher) GetProviders() map[string]Provider { return f.Lda.Providers } func (f *Fetcher) GetHostname() string { return f.Lda.Config.Hostname } func (f *Fetcher) GetIdentityStore() backends.IdentityStorageUpdater { return f.Store } ================================================ FILE: src/backend/protocols/go.imap/imap.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package imap_worker import ( "bytes" "crypto/tls" "encoding/base64" "errors" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/users" log "github.com/Sirupsen/logrus" "github.com/emersion/go-imap" "github.com/emersion/go-imap/client" "github.com/emersion/go-sasl" "strconv" "strings" "time" ) // ImapFetcherHeaders are headers added to emails fetched from remote IMAP type ImapFetcherHeaders map[string]string const ( //gmail related gmail_msgid = "X-GM-MSGID" gmail_labels = "X-GM-LABELS" ) var providers map[string]Provider func init() { // extensions to seek in IMAP server capabilities to identify remote provider providers = map[string]Provider{ // as of april, 2018 (see https://developers.google.com/gmail/imap/imap-extensions) "X-GM-EXT-1": { Name: "gmail", FetchItems: []imap.FetchItem{gmail_msgid, gmail_labels}, }, } } func imapLogin(rId *UserIdentity) (tlsConn *tls.Conn, imapClient *client.Client, provider Provider, err error) { log.Println("Connecting to server...") // Dial TLS directly to be able to dump tls connection state // First try in secure mode, then retry in insecure mode if it fails tlsConn, err = tls.Dial("tcp", rId.Infos["inserver"], nil) if err != nil { log.WithError(err).Warnf("[fetchMail] imapLogin failed to dial tls in secure mode for identity %s (user %s) server : %s", rId.Id, rId.UserId, rId.Infos["inserver"]) log.Infof("[fetchMail] trying INSECURE mode for identity %s (user %s) server : %s", rId.Id, rId.UserId, rId.Infos["inserver"]) tlsConfig := tls.Config{ InsecureSkipVerify: true, } tlsConn, err = tls.Dial("tcp", rId.Infos["inserver"], &tlsConfig) // TODO: save as many data as possible into user_identity table if remote cert is faulty if err != nil { log.WithError(err).Errorf("[fetchMail] imapLogin failed to dial INSECURE tls for identity %s (user %s) server : %s", rId.Id, rId.UserId, rId.Infos["inserver"]) return } } imapClient, err = client.New(tlsConn) if err != nil { log.WithError(err).Error("[fetchMail] imapLogin failed to create IMAP client") return } log.Println("Connected") // identify provider' capabilities capabilities, _ := imapClient.Capability() provider = Provider{Capabilities: capabilities} for capability := range capabilities { if p, ok := providers[capability]; ok { provider.Name = p.Name provider.FetchItems = p.FetchItems } } // choose auth mechanism according to provider capabilities and identity's authtype authType, foundAuthType := rId.Infos["authtype"] if foundAuthType { switch authType { case Oauth1: err = errors.New("oauth1 mechanism not implemented") return case Oauth2: saslClient := sasl.NewXoauth2Client((*rId.Credentials)[users.CRED_USERNAME], (*rId.Credentials)[users.CRED_ACCESS_TOKEN]) err = imapClient.Authenticate(saslClient) if err != nil { log.WithError(err).Errorf("[fetchMail] imapLogin failed to authenticate identity %s with proto Xoauth2", rId.Id) return } case LoginPassword: err = imapClient.Login((*rId.Credentials)["inusername"], (*rId.Credentials)["inpassword"]) if err != nil { log.WithError(err).Errorf("[fetchMail] imapLogin failed to login IMAP for user %s", rId.UserId) return } default: err = fmt.Errorf("unknown auth mechanism : <%s>", authType) return } } else { // fallback by trying default LoginPassword mechanism err = imapClient.Login((*rId.Credentials)["inusername"], (*rId.Credentials)["inpassword"]) if err != nil { log.WithError(err).Error("[fetchMail] imapLogin failed to login IMAP") return } } log.Println("Logged in") return } // syncMailbox will check uidvalidity and fetch only new messages since last sync state saved in RemoteIdentity. // If no previous state found in RemoteIdentity or uidvalidity has changed, syncMailbox will do a full fetch instead. func syncMailbox(ibox *imapBox, imapClient *client.Client, provider Provider, ch chan *imap.Message) (err error) { mbox, err := imapClient.Select(ibox.name, false) if err != nil { log.WithError(err).Errorf("[syncMailbox] failed to select mailbox <%s>", ibox.name) close(ch) return } var from, to uint32 if ibox.lastSync.IsZero() { // first sync, blindly fetch all messages from, to = 1, 0 (*ibox).uidValidity = mbox.UidValidity } else { // check mailbox UIDVALIDITY if ibox.uidValidity != mbox.UidValidity { // TODO // MUST empty the local cache of that mailbox and resync mailbox log.Warnf("[syncMailbox] uidValidity has changed from %d to %d. Local mailbox should resync.", ibox.uidValidity, mbox.UidValidity) // for now, we blindly (re)fetch all messages from, to = 1, 0 (*ibox).uidValidity = mbox.UidValidity } else { if ibox.lastSeenUid == 0 { from, to = 1, 0 } else { from = ibox.lastSeenUid + 1 to = 0 } } } return fetch(imapClient, provider, from, to, ch) } // fetchMailbox retrieves all messages found within remote mailbox // unaware of synchronization func fetchMailbox(ibox *imapBox, imapClient *client.Client, provider Provider, ch chan *imap.Message) (err error) { mbox, err := imapClient.Select(ibox.name, true) if err != nil { log.WithError(err).Error("[fetchMailbox] failed to select INBOX") return } from := uint32(1) to := mbox.UidNext err = fetch(imapClient, provider, from, to, ch) if err != nil { log.WithError(err).Error("[fetchMailbox] failed") } return err } // MashalImap build RFC5322 mail from imap.Message, // adds custom `X-Fetched` headers, // returns an Email suitable to send to our email lda. func MarshalImap(message *imap.Message, xHeaders ImapFetcherHeaders) (mail *Email, err error) { var mailBuff bytes.Buffer for k, v := range xHeaders { mailBuff.WriteString(k + ": " + v + "\r\n") } if len(message.Body) == 1 { // should have only one body for _, body := range message.Body { _, err := mailBuff.ReadFrom(body) if err != nil { //TODO } // stop at first iteration because only one body break } } mail = &Email{ Raw: mailBuff, ImapUid: message.Uid, } return } // buildXheaders builds custom X-Fetched headers // with provider specific information func buildXheaders(tlsConn *tls.Conn, rId *UserIdentity, box *imapBox, message *imap.Message, provider Provider) (xHeaders ImapFetcherHeaders) { connState := tlsConn.ConnectionState() var proto string if provider.Capabilities["IMAP4rev1"] == true { proto = "with IMAP4rev1 protocol" } xHeaders = make(ImapFetcherHeaders) xHeaders["X-Fetched-Imap"] = fmt.Sprintf(`from %s ([%s]) (using %s with cipher %s) by imap-fetcher (Caliopen) %s; %s`, rId.Infos["inserver"], tlsConn.RemoteAddr().String(), TlsVersions[connState.Version], TlsSuites[connState.CipherSuite], proto, time.Now().Format(time.RFC1123Z)) xHeaders["X-Fetched-Imap-Account"] = rId.DisplayName xHeaders["X-Fetched-Imap-Box"] = base64.StdEncoding.EncodeToString([]byte(box.name)) xHeaders["X-Fetched-Imap-For"] = rId.UserId.String() xHeaders["X-Fetched-Imap-Uid"] = strconv.Itoa(int(message.Uid)) if len(message.Flags) > 0 { xHeaders["X-Fetched-Imap-Flags"] = base64.StdEncoding.EncodeToString([]byte(strings.Join(message.Flags, "\r\n"))) } switch provider.Name { case "gmail": if msgid, ok := message.Items[gmail_msgid].(string); ok { xHeaders["X-Fetched-"+gmail_msgid] = msgid } gLabels := strings.Builder{} if labels, ok := message.Items[gmail_labels]; ok && labels != nil { for i, label := range labels.([]interface{}) { if label != nil { if i == 0 { gLabels.WriteString(label.(string)) } else { gLabels.WriteString("\r\n" + label.(string)) } } } } if gLabels.Len() > 0 { xHeaders["X-Fetched-"+gmail_labels] = base64.StdEncoding.EncodeToString([]byte(gLabels.String())) } } return } // from and to must be uid // zero values will be replaced by * wildcard func fetch(imapClient *client.Client, provider Provider, from, to uint32, ch chan *imap.Message) error { if from != 0 && to != 0 && from > to { close(ch) return fmt.Errorf("[fetch] 'to' param is lower than 'from'") } if from != 0 && from == to { log.Info("nothing to fetch") close(ch) return nil } seqset := new(imap.SeqSet) seqset.AddRange(from, to) log.Info("beginning to fetch messages…") items := []imap.FetchItem{imap.FetchFlags, imap.FetchUid, "BODY.PEEK[]"} if len(provider.FetchItems) > 0 { items = append(items, provider.FetchItems...) } return imapClient.UidFetch(seqset, items, ch) } // uploadSentMessage uploads a RFC 5322 mail to relevent `sent` mailbox and flags it has seen func uploadSentMessage(imapClient *client.Client, mail string, date time.Time) error { //1. list mailboxes to find which one is for `sent` messages var sentMbx string boxes := make(chan *imap.MailboxInfo) go func() { err := imapClient.List("", "*", boxes) if err != nil { // ensure channel is closed if _, ok := <-boxes; ok { close(boxes) } } }() var found bool for box := range boxes { if !found { for _, attr := range box.Attributes { if strings.Contains(attr, "Sent") { sentMbx = box.Name found = true break } } if sentMbx == "" { if strings.Contains(box.Name, "Sent") { sentMbx = box.Name found = true } } } } //name still missing, use standard rfc6154#2 if sentMbx == "" { sentMbx = `\Sent` } //2. append mail to mailbox return imapClient.Append(sentMbx, []string{imap.SeenFlag}, date, bytes.NewBufferString(mail)) } ================================================ FILE: src/backend/protocols/go.imap/imap_test.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package imap_worker import ( "crypto/tls" "encoding/base64" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/backendstest" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/helpers" "github.com/emersion/go-imap" "github.com/satori/go.uuid" "reflect" "strings" "testing" "time" ) // NB : as of february'19 IMAP commands are not tested yet func Test_buildXheaders(t *testing.T) { tlsConn := helpers.GetTestTlsConn() userIdentity := &UserIdentity{} box := &imapBox{} message := &imap.Message{} provider := Provider{} type params struct { c *tls.Conn u *UserIdentity i *imapBox m *imap.Message p Provider } connState := tlsConn.ConnectionState() data := []struct { in params out ImapFetcherHeaders }{ { in: params{tlsConn, userIdentity, box, message, provider}, out: ImapFetcherHeaders{ "X-Fetched-Imap-Account": "", "X-Fetched-Imap-Box": "", "X-Fetched-Imap-For": "00000000-0000-0000-0000-000000000000", "X-Fetched-Imap-Uid": "0", "X-Fetched-Imap": fmt.Sprintf(`from %s ([%s]) (using %s with cipher %s) by imap-fetcher (Caliopen) %s; %s`, "", tlsConn.RemoteAddr().String(), TlsVersions[connState.Version], TlsSuites[connState.CipherSuite], "", time.Now().Format(time.RFC1123Z)), }, }, { in: params{tlsConn, userIdentity, box, message, Provider{Capabilities: map[string]bool{"IMAP4rev1": true}}}, out: ImapFetcherHeaders{ "X-Fetched-Imap-Account": "", "X-Fetched-Imap-Box": "", "X-Fetched-Imap-For": "00000000-0000-0000-0000-000000000000", "X-Fetched-Imap-Uid": "0", "X-Fetched-Imap": fmt.Sprintf(`from %s ([%s]) (using %s with cipher %s) by imap-fetcher (Caliopen) %s; %s`, "", tlsConn.RemoteAddr().String(), TlsVersions[connState.Version], TlsSuites[connState.CipherSuite], "with IMAP4rev1 protocol", time.Now().Format(time.RFC1123Z)), }, }, { in: params{tlsConn, &UserIdentity{ DisplayName: "account name", UserId: UUID(uuid.FromStringOrNil(backendstest.EmmaTommeUserId)), }, box, message, provider}, out: ImapFetcherHeaders{ "X-Fetched-Imap-Account": "account name", "X-Fetched-Imap-Box": "", "X-Fetched-Imap-For": uuid.FromStringOrNil(backendstest.EmmaTommeUserId).String(), "X-Fetched-Imap-Uid": "0", "X-Fetched-Imap": fmt.Sprintf(`from %s ([%s]) (using %s with cipher %s) by imap-fetcher (Caliopen) %s; %s`, "", tlsConn.RemoteAddr().String(), TlsVersions[connState.Version], TlsSuites[connState.CipherSuite], "", time.Now().Format(time.RFC1123Z)), }, }, { in: params{tlsConn, userIdentity, &imapBox{name: "box name"}, message, provider}, out: ImapFetcherHeaders{ "X-Fetched-Imap-Account": "", "X-Fetched-Imap-Box": base64.StdEncoding.EncodeToString([]byte("box name")), "X-Fetched-Imap-For": "00000000-0000-0000-0000-000000000000", "X-Fetched-Imap-Uid": "0", "X-Fetched-Imap": fmt.Sprintf(`from %s ([%s]) (using %s with cipher %s) by imap-fetcher (Caliopen) %s; %s`, "", tlsConn.RemoteAddr().String(), TlsVersions[connState.Version], TlsSuites[connState.CipherSuite], "", time.Now().Format(time.RFC1123Z)), }, }, { in: params{tlsConn, userIdentity, box, &imap.Message{Uid: 999}, provider}, out: ImapFetcherHeaders{ "X-Fetched-Imap-Account": "", "X-Fetched-Imap-Box": "", "X-Fetched-Imap-For": "00000000-0000-0000-0000-000000000000", "X-Fetched-Imap-Uid": "999", "X-Fetched-Imap": fmt.Sprintf(`from %s ([%s]) (using %s with cipher %s) by imap-fetcher (Caliopen) %s; %s`, "", tlsConn.RemoteAddr().String(), TlsVersions[connState.Version], TlsSuites[connState.CipherSuite], "", time.Now().Format(time.RFC1123Z)), }, }, { in: params{tlsConn, userIdentity, box, &imap.Message{Flags: []string{"flag1", "flag2"}}, provider}, out: ImapFetcherHeaders{ "X-Fetched-Imap-Account": "", "X-Fetched-Imap-Box": "", "X-Fetched-Imap-For": "00000000-0000-0000-0000-000000000000", "X-Fetched-Imap-Uid": "0", "X-Fetched-Imap-Flags": base64.StdEncoding.EncodeToString([]byte("flag1\r\nflag2")), "X-Fetched-Imap": fmt.Sprintf(`from %s ([%s]) (using %s with cipher %s) by imap-fetcher (Caliopen) %s; %s`, "", tlsConn.RemoteAddr().String(), TlsVersions[connState.Version], TlsSuites[connState.CipherSuite], "", time.Now().Format(time.RFC1123Z)), }, }, { in: params{tlsConn, userIdentity, box, message, Provider{Name: "gmail"}}, out: ImapFetcherHeaders{ "X-Fetched-Imap-Account": "", "X-Fetched-Imap-Box": "", "X-Fetched-Imap-For": "00000000-0000-0000-0000-000000000000", "X-Fetched-Imap-Uid": "0", "X-Fetched-Imap": fmt.Sprintf(`from %s ([%s]) (using %s with cipher %s) by imap-fetcher (Caliopen) %s; %s`, "", tlsConn.RemoteAddr().String(), TlsVersions[connState.Version], TlsSuites[connState.CipherSuite], "", time.Now().Format(time.RFC1123Z)), }, }, { in: params{tlsConn, userIdentity, box, &imap.Message{Items: map[imap.FetchItem]interface{}{ imap.FetchItem(gmail_msgid): "gmail message-id", imap.FetchItem(gmail_labels): []interface{}{`\Inbox`, `Important`, `Très Important`}, }}, Provider{Name: "gmail"}}, out: ImapFetcherHeaders{ "X-Fetched-Imap-Account": "", "X-Fetched-Imap-Box": "", "X-Fetched-Imap-For": "00000000-0000-0000-0000-000000000000", "X-Fetched-Imap-Uid": "0", "X-Fetched-" + gmail_msgid: "gmail message-id", "X-Fetched-" + gmail_labels: "XEluYm94DQpJbXBvcnRhbnQNClRyw6hzIEltcG9ydGFudA==", "X-Fetched-Imap": fmt.Sprintf(`from %s ([%s]) (using %s with cipher %s) by imap-fetcher (Caliopen) %s; %s`, "", tlsConn.RemoteAddr().String(), TlsVersions[connState.Version], TlsSuites[connState.CipherSuite], "", time.Now().Format(time.RFC1123Z)), }, }, } for i, set := range data { result := buildXheaders(set.in.c, set.in.u, set.in.i, set.in.m, set.in.p) if !reflect.DeepEqual(result, set.out) { t.Errorf("invalid headers for set %d.\nExpected : %+v\nGot : %+v", i, set.out, result) } } } func TestMarshalImap(t *testing.T) { message := imap.NewMessage(42, []imap.FetchItem{imap.FetchBody, imap.FetchFlags}) const mailBody = "mail body" const headerKey = "x-test-header" const headerValue = "test-header-value" message.Body = map[*imap.BodySectionName]imap.Literal{&imap.BodySectionName{}: strings.NewReader(mailBody)} xHeaders := map[string]string{headerKey: headerValue} email, err := MarshalImap(message, xHeaders) if err != nil { t.Error(err) } // expect xHeaders added at head of email's bytes.Buffer s := headerKey + ": " + headerValue + "\r\n" + mailBody if s != email.Raw.String() { t.Errorf("MarshalImap failed to build expected email, expected :\n%s\ngot :\n%s\n", s, email.Raw.String()) } } ================================================ FILE: src/backend/protocols/go.imap/lda.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package imap_worker import ( "errors" "fmt" broker "github.com/CaliOpen/Caliopen/src/backend/brokers/go.emails" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/Notifications" "github.com/satori/go.uuid" "time" ) //Local Delivery Agent, in charge of IO between fetcher and our email broker type Lda struct { Config WorkerConfig broker *broker.EmailBroker brokerConnectors broker.EmailBrokerConnectors Providers map[string]Provider } func NewLda(config WorkerConfig) (*Lda, error) { var err error lda := Lda{} lda.Config = config lda.broker, lda.brokerConnectors, err = broker.Initialize(config.LDAConfig) lda.Providers = make(map[string]Provider) for _, provider := range config.LDAConfig.Providers { lda.Providers[provider.Name] = provider } return &lda, err } func (lda *Lda) shutdown() error { lda.broker.ShutDown() return nil } func (lda *Lda) deliverMail(mail *Email, userId, identityID string, batch *Notifications.BatchNotification) (err error) { emailMsg := &EmailMessage{ Email: mail, Message: &Message{ User_id: UUID(uuid.FromStringOrNil(userId)), UserIdentities: []UUID{UUID(uuid.FromStringOrNil(identityID))}, }, } incoming := &broker.SmtpEmail{ EmailMessage: emailMsg, Response: make(chan *broker.EmailDeliveryAck), Batch: batch, } defer close(incoming.Response) lda.brokerConnectors.Ingress <- incoming select { case response := <-incoming.Response: if response.Err { return errors.New(fmt.Sprintf("[deliverMail] Error : " + response.Response)) } return nil case <-time.After(30 * time.Second): return errors.New("[deliverMail] LDA timeout") } } ================================================ FILE: src/backend/protocols/go.imap/sender.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package imap_worker import ( "encoding/json" "errors" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/users" log "github.com/Sirupsen/logrus" "github.com/nats-io/go-nats" "time" ) type Sender struct { Hostname string ImapProviders map[string]Provider NatsConn *nats.Conn NatsMessage *nats.Msg OutSMTPtopic string Store backends.LDAStore } // unexported vars to help override funcs in tests var ( sendDraft = func(s *Sender, msg *nats.Msg) { s.SendDraft(msg) } uploadSentMessageToRemote = func(s *Sender, userIdentity *UserIdentity, msg *Message) error { return s.UploadSentMessageToRemote(userIdentity, msg) } ) func (s *Sender) SendDraft(msg *nats.Msg) { var order BrokerOrder err := json.Unmarshal(msg.Data, &order) if err != nil { s.natsReplyError(msg, fmt.Errorf("Unable to unmarshal message from NATS. Payload was <%s>", string(msg.Data))) return } // get userIdentity and check auth params validity if err != nil { s.natsReplyError(msg, err) return } userIdentity, err := s.Store.RetrieveUserIdentity(order.UserId, order.IdentityId, true) if err != nil { s.natsReplyError(msg, err) return } if userIdentity.Infos["authtype"] == Oauth2 { err = users.ValidateOauth2Credentials(userIdentity, s, true) if err != nil { s.natsReplyError(msg, err) return } } //1. make use of our lmtpd to send email natsMessage, e := json.Marshal(order) if e != nil { s.natsReplyError(msg, errors.New("[SendDraft] failed to build nats message")) return } smtpReply, err := s.NatsConn.Request(s.OutSMTPtopic, []byte(natsMessage), 30*time.Second) //2. handle LMTP response if err != nil { if smtpReply != nil { s.natsReplyError(msg, errors.New(smtpReply.Reply)) return } else { s.natsReplyError(msg, err) return } } var reply DeliveryAck err = json.Unmarshal(smtpReply.Data, &reply) if err != nil { s.natsReplyError(msg, fmt.Errorf("[IMAPworker]SendDraft failed to unmarshal smtpReply : %s", err)) return } if reply.Err { s.natsReplyError(msg, errors.New(reply.Response)) return } //3. no error when sending email, // if applicable upload a copy to remote IMAP account if userIdentity.Type == RemoteIdentity { sentMsg, err := s.Store.RetrieveMessage(order.UserId, order.MessageId) if err != nil { s.natsReplyError(msg, fmt.Errorf("[IMAPworker]SendDraft failed to retrieve sent message : %s", err)) return } err = uploadSentMessageToRemote(s, userIdentity, sentMsg) if err != nil { s.natsReplyError(msg, fmt.Errorf("[IMAPworker]SendDraft failed to upload sent email to remote IMAP account : %s", err)) return } } //4. respond to caller _ = s.NatsConn.Publish(msg.Reply, smtpReply.Data) } func (s *Sender) natsReplyError(msg *nats.Msg, err error) { log.WithError(err).Warnf("IMAPworker [outbound] : error when processing incoming nats message : %+v", *msg) ack := DeliveryAck{ Err: true, Response: fmt.Sprintf("failed to handle order with error « %s » ", err.Error()), } json_resp, _ := json.Marshal(ack) s.NatsConn.Publish(msg.Reply, json_resp) } func (s *Sender) UploadSentMessageToRemote(userIdentity *UserIdentity, msg *Message) error { //get raw message rawMail, err := s.Store.GetRawMessage(msg.Raw_msg_id.String()) if err != nil { return err } if userIdentity.Infos["authtype"] == Oauth2 { err = users.ValidateOauth2Credentials(userIdentity, s, true) if err != nil { return err } } _, imapClient, _, err := imapLogin(userIdentity) if err != nil { return err } defer imapClient.Logout() return uploadSentMessage(imapClient, rawMail.Raw_data, msg.Date) } /* Oauth2Interfacer implementation */ func (s *Sender) GetProviders() map[string]Provider { return s.ImapProviders } func (s *Sender) GetHostname() string { return s.Hostname } func (s *Sender) GetIdentityStore() backends.IdentityStorageUpdater { return s.Store } ================================================ FILE: src/backend/protocols/go.imap/sender_test.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package imap_worker import ( "encoding/json" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/backendstest" "github.com/nats-io/gnatsd/server" "github.com/nats-io/go-nats" "github.com/satori/go.uuid" "testing" "time" ) const ( replyErrorTopic = "testReplyError" ) func initTestSender() (sender *Sender, natsServer *server.Server, err error) { worker, natsServer, err := newWorkerTest() sender = &Sender{ Hostname: worker.Config.Hostname, ImapProviders: worker.Lda.Providers, NatsConn: worker.NatsConn, OutSMTPtopic: worker.Config.LDAConfig.OutTopic, Store: worker.Store, } return } // the whole process of sending an email is not tested here, // but only APIs calls and responses/errors handling func TestSender_SendDraft(t *testing.T) { sender, natsServer, err := initTestSender() defer natsServer.Shutdown() if err != nil { t.Error(err) return } c := make(chan struct{}) // add a global subscriber to test errors replies globalErrSub, err := sender.NatsConn.Subscribe(replyErrorTopic, func(msg *nats.Msg) { defer close(c) var resp DeliveryAck err := json.Unmarshal(msg.Data, &resp) if err != nil { t.Error(err) return } if !resp.Err { t.Error("expected DeliveryAck.Err == true, got false") return } if resp.Response == "" { t.Error("expected DeliveryAck.Response to be non empty string, got empty string") } }) if err != nil { t.Error(err) return } // test SendDraft with non existent identity fakeUUID := uuid.NewV4().String() sendOrder := BrokerOrder{ IdentityId: fakeUUID, MessageId: fakeUUID, Order: "deliver", UserId: fakeUUID, } jsonOrder, _ := json.Marshal(sendOrder) natsPayload := nats.Msg{ Subject: "test", Reply: replyErrorTopic, Data: jsonOrder, } sender.SendDraft(&natsPayload) select { case <-c: case <-time.After(time.Second): t.Errorf("timeout waiting for sendDraft response for order : %+v", sendOrder) } // test SendDraft with LMTP responding an error c = make(chan struct{}) lmtpErrorSub, err := sender.NatsConn.Subscribe(sender.OutSMTPtopic, func(msg *nats.Msg) { err := sender.NatsConn.Publish(msg.Reply, []byte(`{"error":true,"message":"fake smtp error"}`)) if err != nil { t.Error(err) } }) if err != nil { t.Error(err) } else { sendOrder := BrokerOrder{ IdentityId: "7e4eb26d-1b70-4bb3-b556-6c54f046e88e", MessageId: fakeUUID, Order: "deliver", UserId: backendstest.EmmaTommeUserId, } jsonOrder, _ := json.Marshal(sendOrder) natsPayload := nats.Msg{ Subject: "test", Reply: replyErrorTopic, Data: jsonOrder, } sender.SendDraft(&natsPayload) select { case <-c: case <-time.After(time.Second): t.Errorf("timeout waiting for sendDraft response for order : %+v", sendOrder) } } _ = lmtpErrorSub.Unsubscribe() // test SendDraft with valid payload and OK from lmtp // but invalid message ID c = make(chan struct{}) lmtpBadMsgSub, err := sender.NatsConn.Subscribe(sender.OutSMTPtopic, func(msg *nats.Msg) { err := sender.NatsConn.Publish(msg.Reply, []byte(`{"error":false,"message":""}`)) if err != nil { t.Error(err) } }) if err != nil { t.Error(err) } else { sendOrder := BrokerOrder{ IdentityId: "7e4eb26d-1b70-4bb3-b556-6c54f046e88e", MessageId: fakeUUID, Order: "deliver", UserId: backendstest.EmmaTommeUserId, } jsonOrder, _ := json.Marshal(sendOrder) natsPayload := nats.Msg{ Subject: "test", Reply: replyErrorTopic, Data: jsonOrder, } sender.SendDraft(&natsPayload) select { case <-c: case <-time.After(time.Second): t.Errorf("timeout waiting for sendDraft response for order : %+v", sendOrder) } } _ = lmtpBadMsgSub.Unsubscribe() // test SendDraft with valid payload and OK from lmtp // should call uploadSentMessageToRemote because identity is a remote one // and should re-publish lmtp reply _ = globalErrSub.Unsubscribe() c = make(chan struct{}) lmtpOKMsgSub, err := sender.NatsConn.Subscribe(sender.OutSMTPtopic, func(msg *nats.Msg) { err := sender.NatsConn.Publish(msg.Reply, []byte(`{"error":false,"message":""}`)) if err != nil { t.Error(err) } }) _, err = sender.NatsConn.Subscribe("ok reply", func(msg *nats.Msg) { defer close(c) var resp DeliveryAck err := json.Unmarshal(msg.Data, &resp) if err != nil { t.Error(err) return } if resp.Err { t.Error("expected DeliveryAck.Err == false, got true") return } if resp.Response != "" { t.Error("expected DeliveryAck.Response to be empty string, got non empty string") } }) if err != nil { t.Error(err) } else { sendOrder := BrokerOrder{ IdentityId: "7e4eb26d-1b70-4bb3-b556-6c54f046e88e", MessageId: "b26e5ba4-34cc-42bb-9b70-5279648134f8", Order: "deliver", UserId: backendstest.EmmaTommeUserId, } jsonOrder, _ := json.Marshal(sendOrder) natsPayload := nats.Msg{ Subject: "test", Reply: "ok reply", Data: jsonOrder, } // override func to prevent cascading calls uploadSentMessageToRemote = func(s *Sender, userIdentity *UserIdentity, msg *Message) error { return nil } sender.SendDraft(&natsPayload) select { case <-c: case <-time.After(time.Second): t.Errorf("timeout waiting for sendDraft response for order : %+v", sendOrder) } } _ = lmtpOKMsgSub.Unsubscribe() } ================================================ FILE: src/backend/protocols/go.imap/worker.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package imap_worker import ( "encoding/json" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/cassandra" log "github.com/Sirupsen/logrus" "github.com/gocql/gocql" "github.com/nats-io/go-nats" "sync" "time" ) type Worker struct { Config WorkerConfig Id string Lda *Lda NatsConn *nats.Conn NatsSubs []*nats.Subscription Store backends.LDAStore HaltGroup *sync.WaitGroup } const ( noPendingJobErr = "no pending job" needJobOrderStr = `{"worker":"%s","order":{"order":"need_job"}}` ) // NewWorker loads config, checks for errors then returns a worker ready to start. func NewWorker(config WorkerConfig, id string) (worker *Worker, err error) { w := Worker{ Config: config, Id: id, NatsSubs: make([]*nats.Subscription, 1), } //copy relevant config to LDAConfig w.Config.LDAConfig.StoreName = w.Config.StoreName w.Config.LDAConfig.NatsURL = w.Config.NatsUrl w.Config.LDAConfig.StoreConfig = w.Config.StoreConfig // init all connexions, they will be pass to fetchers // Lda w.Lda, err = NewLda(w.Config) if err != nil { log.WithError(err).Warn("[NewWorker] : initalization of LDA failed") return nil, err } // Nats w.NatsConn, err = nats.Connect(config.NatsUrl) if err != nil { log.WithError(err).Warn("[NewWorker] : initalization of NATS connexion failed") return nil, err } // Store switch config.StoreName { case "cassandra": c := store.CassandraConfig{ Hosts: config.StoreConfig.Hosts, Keyspace: config.StoreConfig.Keyspace, Consistency: gocql.Consistency(config.StoreConfig.Consistency), SizeLimit: config.StoreConfig.SizeLimit, UseVault: config.StoreConfig.UseVault, } if config.StoreConfig.ObjectStore == "s3" { c.WithObjStore = true c.Endpoint = config.StoreConfig.OSSConfig.Endpoint c.AccessKey = config.StoreConfig.OSSConfig.AccessKey c.SecretKey = config.StoreConfig.OSSConfig.SecretKey c.RawMsgBucket = config.StoreConfig.OSSConfig.Buckets["raw_messages"] c.AttachmentBucket = config.StoreConfig.OSSConfig.Buckets["temporary_attachments"] c.Location = config.StoreConfig.OSSConfig.Location } if c.UseVault { c.Url = config.StoreConfig.VaultConfig.Url c.Username = config.StoreConfig.VaultConfig.Username c.Password = config.StoreConfig.VaultConfig.Password } w.Store, err = store.InitializeCassandraBackend(c) if err != nil { log.WithError(err).Warnf("[NewWorker] initalization of %s backend failed", config.StoreName) return nil, err } } return &w, nil } func (worker *Worker) Start(throttling ...time.Duration) error { var throttle time.Duration if len(throttling) == 1 && throttling[0] != 0 { throttle = throttling[0] } else { throttle = pollThrottling } var err error (*worker).NatsSubs[0], err = worker.NatsConn.QueueSubscribe(worker.Config.NatsTopicSender, worker.Config.NatsQueue, worker.natsMsgHandler) if err != nil { return err } worker.NatsConn.Flush() log.Infof("IMAP worker %s starting with %d sec throttling", worker.Id, throttle/time.Second) // start throttled jobs polling for { start := time.Now() requestOrder := []byte(fmt.Sprintf(needJobOrderStr, worker.Id)) log.Infof("IMAP worker %s is requesting jobs to idpoller", worker.Id) resp, err := worker.NatsConn.Request(worker.Config.NatsTopicPoller, requestOrder, time.Minute) if err != nil { log.WithError(err).Warnf("[worker %s] failed to request pending jobs on nats", worker.Id) } else { worker.natsMsgHandler(resp) } // check for interrupt after job is finished if worker.HaltGroup != nil { worker.Stop() break } elapsed := time.Now().Sub(start) if elapsed < throttle { time.Sleep(throttle - elapsed) } } return nil } func (worker *Worker) Stop() { for _, sub := range worker.NatsSubs { sub.Unsubscribe() } worker.NatsConn.Close() worker.Store.Close() worker.Lda.broker.Store.Close() worker.Lda.broker.NatsConn.Close() worker.HaltGroup.Done() log.Infof("worker %s stopped", worker.Id) } // MsgHandler parses message and launches appropriate goroutine to handle requested operations func (worker *Worker) natsMsgHandler(msg *nats.Msg) { message := IMAPorder{} err := json.Unmarshal(msg.Data, &message) if err != nil { log.WithError(err).Errorf("Unable to unmarshal message from NATS. Payload was <%s>", string(msg.Data)) return } switch message.Order { case noPendingJobErr: return case "sync": // simplest order to initiate a sync op for a stored remote identity fetcher := Fetcher{ Hostname: worker.Config.Hostname, Lda: worker.Lda, Store: worker.Store, } syncRemoteWithLocal(&fetcher, message) case "fullfetch": // order sent by imapctl to initiate a fetch op for an user fetcher := Fetcher{ Hostname: worker.Config.Hostname, Lda: worker.Lda, Store: worker.Store, } fetchRemoteToLocal(&fetcher, message) case "deliver": // order sent by api2 to send a draft via remote SMTP/IMAP sender := Sender{ Hostname: worker.Config.Hostname, ImapProviders: worker.Lda.Providers, NatsConn: worker.NatsConn, NatsMessage: msg, OutSMTPtopic: worker.Config.LDAConfig.OutTopic, Store: worker.Store, } sendDraft(&sender, msg) case "test": log.Info("Order « test » received") } } ================================================ FILE: src/backend/protocols/go.imap/worker_test.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package imap_worker import ( "encoding/json" "fmt" "github.com/CaliOpen/Caliopen/src/backend/brokers/go.emails" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/NATS/go.mockednats" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/backendstest" "github.com/nats-io/gnatsd/server" "github.com/nats-io/go-nats" "sync" "testing" "time" ) const ( natsUrl = "0.0.0.0" ) func newWorkerTest() (worker *Worker, natsServer *server.Server, err error) { worker = &Worker{ Config: WorkerConfig{ LDAConfig: LDAConfig{ OutTopic: "outboundIMAP", }, NatsQueue: "IMAPworkers", NatsTopicPoller: "imapJobs", NatsTopicPollerCache: "idCache", NatsTopicSender: "outboundIMAP", }, Id: "testWorker", NatsSubs: make([]*nats.Subscription, 1), } natsServer, natsConn, err := mockednats.GetNats() if err != nil { return nil, nil, err } worker.NatsConn = natsConn connectors := email_broker.EmailBrokerConnectors{ Ingress: make(chan *email_broker.SmtpEmail), Egress: make(chan *email_broker.SmtpEmail), } worker.Lda = &Lda{ broker: &email_broker.EmailBroker{ Store: backendstest.GetLDAStoreBackend(), Index: backendstest.GetLDAIndexBackend(), NatsConn: worker.NatsConn, Connectors: connectors, }, brokerConnectors: connectors, Providers: make(map[string]Provider), } if err != nil { return nil, nil, fmt.Errorf("[initMqHandler] failed to init NATS connection : %s", err) } worker.Store = backendstest.GetLDAStoreBackend() return } func TestWorker_StartAndStop(t *testing.T) { w, s, err := newWorkerTest() if err != nil { t.Error(err) } defer s.Shutdown() // test if worker requests on Nats every second with the right payload c := make(chan struct{}) wg := new(sync.WaitGroup) wg.Add(1) go w.Start(time.Second) count := 0 _, err = w.NatsConn.Subscribe("imapJobs", func(msg *nats.Msg) { var req WorkerRequest err := json.Unmarshal(msg.Data, &req) if err != nil { t.Errorf("unable to unmarshal worker's request : %s", err) return } if req.Order.Order != "need_job" { t.Errorf("expected to receive order 'need_job', got %s", req.Order.Order) } w.NatsConn.Publish(msg.Reply, []byte(`{"order":"no pending job"}`)) count++ if count == 3 { wg.Done() return } }) if err != nil { t.Error(err) } go func() { wg.Wait() close(c) }() select { case <-c: // worker confirmed to send request every second ; now test halting w.HaltGroup = new(sync.WaitGroup) w.HaltGroup.Add(1) time.Sleep(500 * time.Millisecond) if !w.NatsConn.IsClosed() { t.Error("expected worker's nats connexion to be closed") } for _, sub := range w.NatsSubs { if sub.IsValid() { t.Errorf("expected all worker's subscription closed, got <%s> still valid", sub.Subject) } } return case <-time.After(5 * time.Second): t.Error("timeout waiting for worker to send requests on nats") } } func TestWorker_natsMsgHandler(t *testing.T) { w, s, err := newWorkerTest() if err != nil { t.Error(err) } defer s.Shutdown() c := make(chan struct{}) // overriding funcs that should be called within natsMsgHandler but are out of this test scope syncRemoteWithLocal = func(f *Fetcher, order IMAPorder) error { defer close(c) if f == nil { t.Error("expected a Fetcher within syncRemoteWithLocal call, got nil") return nil } if f.Store != w.Store { t.Errorf("expected a fetcher set with worker's store, got %+v", f.Store) } if f.Lda != w.Lda { t.Errorf("expected a fetcher set with worker's store, got %+v", f.Store) } return nil } fetchRemoteToLocal = func(f *Fetcher, order IMAPorder) error { defer close(c) if f == nil { t.Error("expected a Fetcher within fetchRemoteToLocal call, got nil") return nil } if f.Store != w.Store { t.Errorf("expected a fetcher set with worker's store, got %+v", f.Store) } if f.Lda != w.Lda { t.Errorf("expected a fetcher set with worker's store, got %+v", f.Store) } return nil } sendDraft = func(s *Sender, msg *nats.Msg) { defer close(c) if s == nil { t.Error("expected a Sender within sendDraft call, got nil") return } if s.Store != w.Store { t.Errorf("expected a sender set with worker's store, got %+v", s.Store) } if s.NatsConn != w.NatsConn { t.Errorf("expected a sender set with worker's natsConn, got %+v", s.NatsConn) } if s.NatsMessage != msg { t.Errorf("expected a sender with nats message embedded, got %+v", s.NatsMessage) } } // test orders handling // 'sync' order := IMAPorder{ Order: "sync", } data, _ := json.Marshal(order) natsPayload := nats.Msg{ Subject: "test", Reply: "testMsgReply", Data: data, } w.natsMsgHandler(&natsPayload) select { case <-c: case <-time.After(10 * time.Millisecond): t.Error("expected 'sync' order to trigger a call to syncRemoteWithLocal func, but func was not called") } // 'fullfetch' c = make(chan struct{}) order.Order = "fullfetch" data, _ = json.Marshal(order) natsPayload = nats.Msg{ Subject: "test", Reply: "testMsgReply", Data: data, } w.natsMsgHandler(&natsPayload) select { case <-c: case <-time.After(10 * time.Millisecond): t.Error("expected 'fullfetch' order to trigger a call to fetchRemoteToLocal func, but func was not called") } // 'deliver' c = make(chan struct{}) order.Order = "deliver" data, _ = json.Marshal(order) natsPayload = nats.Msg{ Subject: "test", Reply: "testMsgReply", Data: data, } w.natsMsgHandler(&natsPayload) select { case <-c: case <-time.After(10 * time.Millisecond): t.Error("expected 'deliver' order to trigger a call to sendDraft func, but func was not called") } } ================================================ FILE: src/backend/protocols/go.mastodon/account.go ================================================ // Copyleft (ɔ) 2018 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package mastodonworker import ( "bytes" "encoding/json" "errors" "fmt" "strconv" "time" "context" broker "github.com/CaliOpen/Caliopen/src/backend/brokers/go.mastodon" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/Notifications" log "github.com/Sirupsen/logrus" "github.com/mattn/go-mastodon" "strings" ) type ( AccountHandler struct { AccountDesk chan uint broker *broker.MastodonBroker closed bool lastDMseen string MasterDesk chan DeskMessage mastodonClient *mastodon.Client userAccount *MastodonAccount } MastodonAccount struct { displayName string mastodonID string remoteID UUID userID UUID username string } ) const ( //AccountDesk commands PollDM = uint(iota) PollTimeLine Stop lastSeenInfosKey = "lastseendm" lastSyncInfosKey = "lastsync" lastErrorKey = "lastFetchError" dateFirstErrorKey = "firstErrorDate" dateLastErrorKey = "lastErrorDate" errorsCountKey = "errorsCount" syncingKey = "syncing" defaultPollInterval = 10 syncingTimeout = 1 // how many hours to wait before restarting sync op ) // NewAccountHandler creates a handler dedicated to a specific mastodon account. // It caches remote identity credentials and data, as well as user context connection to mastodon API. func NewAccountHandler(userID, remoteID string, worker Worker) (accountHandler *AccountHandler, err error) { accountHandler = new(AccountHandler) accountHandler.AccountDesk = make(chan uint) accountHandler.MasterDesk = worker.Desk b, e := broker.Initialize(worker.Conf.BrokerConfig, worker.Store, worker.Index, worker.NatsConn, worker.Notifier) if e != nil { err = fmt.Errorf("[MastodonAccount]NewAccountHandler failed to initialize a mastodon broker : %s", e) return nil, err } accountHandler.broker = b var remote *UserIdentity // retrieve data from db remote, err = accountHandler.broker.Store.RetrieveUserIdentity(userID, remoteID, true) if err != nil { log.WithError(err).Errorf("[MastodonAccount]NewAccountHandler failed to retrieve remote identity <%s> (user <%s>)", remoteID, userID) return } if remote.Credentials == nil { err = fmt.Errorf("[MastodonAccount]NewAccountHandler failed to retrieve credentials for remote identity <%s> (user <%s>)", remoteID, userID) return } userAcct := strings.Split(remote.Identifier, "@") if len(userAcct) != 2 { err = fmt.Errorf("[MastodonAccount]NewAccountHandler failed to split user account identifier : <%s>", remote.Identifier) return } provider, e := accountHandler.broker.Store.RetrieveProvider("mastodon", userAcct[1]) if e != nil { log.WithError(e) err = fmt.Errorf("[MastodonAccount]NewAccountHandler failed to retrieve provider %s from db : %s", userAcct[1], e) return } accountHandler.userAccount = &MastodonAccount{ displayName: remote.DisplayName, mastodonID: remote.Infos["mastodon_id"], remoteID: remote.Id, userID: remote.UserId, username: userAcct[0], } if lastseen, ok := remote.Infos[lastSeenInfosKey]; ok { accountHandler.lastDMseen = lastseen } else { accountHandler.lastDMseen = "0" } accountHandler.mastodonClient = mastodon.NewClient(&mastodon.Config{ AccessToken: (*remote.Credentials)["oauth2accesstoken"], ClientID: provider.Infos["client_id"], ClientSecret: provider.Infos["client_secret"], Server: provider.Infos["address"], }) return } // Start begins infinite loops, until receiving stop order. This func must be call within goroutine. func (worker *AccountHandler) Start() { go func(w *AccountHandler) { for worker.broker != nil { select { case egress, ok := <-w.broker.Connectors.Egress: if !ok { if !worker.closed { close(worker.broker.Connectors.Halt) close(worker.AccountDesk) worker.closed = true } worker.MasterDesk <- DeskMessage{closeAccountOrder, worker} return } err := w.SendDM(egress.Order) if err != nil { egress.Ack <- &DeliveryAck{ Err: true, Response: err.Error(), } } else { egress.Ack <- &DeliveryAck{ Err: false, Response: "OK", } } case <-w.broker.Connectors.Halt: worker.MasterDesk <- DeskMessage{closeAccountOrder, worker} return } } }(worker) for command := range worker.AccountDesk { switch command { case PollDM: worker.PollDM() case Stop: worker.Stop() return default: log.Warnf("worker received unknown command number %d", command) } } } func (worker *AccountHandler) Stop() { if !worker.closed { close(worker.broker.Connectors.Egress) close(worker.broker.Connectors.Halt) close(worker.AccountDesk) worker.closed = true } } // PollDM calls Mastodon API endpoint to fetch DMs // it passes unseen DM to its embedded broker func (worker *AccountHandler) PollDM() { // retrieve user_identity.infos accountInfos, retrieveErr := worker.broker.Store.RetrieveRemoteInfosMap(worker.userAccount.userID.String(), worker.userAccount.remoteID.String()) if retrieveErr != nil { log.WithError(retrieveErr).Warnf("[AccountHandler %s] PollDM failed to retrieve infos map", worker.userAccount.remoteID.String()) return } // check if a sync process is running if syncing, ok := accountInfos[syncingKey]; ok && syncing != "" { startDate, e := time.Parse(time.RFC3339, syncing) if e == nil && time.Since(startDate)/time.Hour < syncingTimeout { log.Infof("[PollDM] avoiding concurrent sync for <%s>. Syncing in progress since %s", worker.userAccount.remoteID, accountInfos["syncing"]) return } } // save syncing state in db to prevent concurrent sync accountInfos[syncingKey] = time.Now().Format(time.RFC3339) err := worker.broker.Store.UpdateRemoteInfosMap(worker.userAccount.userID.String(), worker.userAccount.remoteID.String(), accountInfos) if err != nil { log.WithError(err).Infof("[PollDM] failed to update syncing state user <%s>, identity <%s>", worker.userAccount.userID, worker.userAccount.remoteID) return } // do not forget to always write down last_check timestamp // and to remove syncing state before leaving defer func() { if worker.broker != nil { delete(accountInfos, syncingKey) e := worker.broker.Store.UpdateRemoteInfosMap(worker.userAccount.userID.String(), worker.userAccount.remoteID.String(), accountInfos) if e != nil { log.WithError(e).Warnf("[AccountHandler %s] PollDM failed to update sync state in db", worker.userAccount.remoteID.String()) } log.Infof("[AccountHandler %s] PollDM finished", worker.userAccount.remoteID.String()) e = worker.broker.Store.TimestampRemoteLastCheck(worker.userAccount.userID.String(), worker.userAccount.remoteID.String()) if e != nil { log.WithError(e).Warnf("[AccountHandler %s] PollDM failed to update last_check state in db", worker.userAccount.remoteID.String()) } } }() // retrieve DM list from mastodon API // 40 statuses by 40 statuses in reverse order // until lastSeenDM or end of feed pg := &mastodon.Pagination{} statuses := make([]*mastodon.Status, 0, 40) var getErr error for { pg.Limit = 40 pg.SinceID = "" pg.MinID = "" page, e := worker.mastodonClient.GetTimelineDirect(context.Background(), pg) // GetTimelineDirect will update pg.maxID if e != nil { getErr = e break } if len(page) == 0 { break } statuses = append(statuses, page...) if accountInfos[lastSeenInfosKey] != "" && broker.IDgreaterOrEqual(accountInfos[lastSeenInfosKey], string(pg.MaxID)) { break } } if getErr != nil { log.WithError(getErr) e := worker.saveErrorState(accountInfos, getErr.Error()) if e != nil { log.WithError(e).Warnf("[AccountHandler %s] PollDM failed to update sync state in db", worker.userAccount.remoteID.String()) } return } // inject DM in Caliopen, reverse order batch := Notifications.NewBatch("mastodon_worker") for i := len(statuses) - 1; i >= 0; i-- { if broker.IDgreaterOrEqual(string(statuses[i].ID), accountInfos[lastSeenInfosKey]) && string(statuses[i].ID) != accountInfos[lastSeenInfosKey] { processErr := worker.broker.ProcessInDM(worker.userAccount.userID, worker.userAccount.remoteID, statuses[i], true, batch) if processErr != nil { log.WithError(processErr).Warnf("[AccountHandler %s] ProcessInDM failed for status: %+v", worker.userAccount.remoteID.String(), statuses[i]) } else { accountInfos[lastSeenInfosKey] = string(statuses[i].ID) } } } accountInfos[lastSyncInfosKey] = time.Now().Format(time.RFC3339) // sync terminated without error, cleanup userIdentity infos map delete(accountInfos, lastErrorKey) delete(accountInfos, errorsCountKey) delete(accountInfos, dateFirstErrorKey) delete(accountInfos, dateLastErrorKey) err = worker.broker.Store.UpdateRemoteInfosMap(worker.userAccount.userID.String(), worker.userAccount.remoteID.String(), accountInfos) if err != nil { log.WithError(err).Warnf("[AccountHandler %s] ProcessInDM failed to update InfosMap at end of process", worker.userAccount.remoteID.String()) } // notify new messages batch.Save(worker.broker.Notifier, "", LongLived) } func (worker *AccountHandler) dmNotSeen(status mastodon.Status) bool { return worker.lastDMseen < string(status.ID) } // SendDM delivers DM to Mastodon endpoint and give back Mastodon's response to broker. func (worker *AccountHandler) SendDM(order BrokerOrder) error { // make use of broker to marshal a direct message brokerPort := make(chan *broker.DMpayload) var brokerMessage *broker.DMpayload go worker.broker.ProcessOutDM(order, brokerPort) select { case brokerMessage = <-brokerPort: if brokerMessage.Err != nil { return brokerMessage.Err } case <-time.After(10 * time.Second): return errors.New("[SendDM] broker timeout") } // deliver DM through Mastodon API status, errResponse := worker.mastodonClient.PostStatus(context.Background(), brokerMessage.Toot) if errResponse != nil { brokerMessage.Response <- broker.MastodonDeliveryAck{ Payload: status, Err: true, Response: errResponse.Error(), } return errResponse } // give back Mastodon's reply to broker for it finishes its job brokerMessage.Response <- broker.MastodonDeliveryAck{ Payload: status, Err: false, } select { case brokerMessage = <-brokerPort: if brokerMessage.Err != nil { return brokerMessage.Err } return nil case <-time.After(10 * time.Second): return errors.New("[SendDM] broker timeout") } } // getAccountName returns Mastodon account screen name given a Mastodon account ID // screen name is retrieve either from worker's cache or Mastodon API // returns empty string if it fails. func (worker *AccountHandler) getAccountName(accountID string) (accountName string) { // useless ? return "" } // isDMUnique returns true if Mastodon Direct Message id is not found within user's messages index // if seeking fails for any reason, true is returned anyway to allow duplication func (worker *AccountHandler) isDMUnique(dmID string) bool { messageID, err := worker.broker.Store.SeekMessageByExternalRef(worker.userAccount.userID.String(), dmID, "") if err != nil || bytes.Equal(messageID.Bytes(), EmptyUUID.Bytes()) { return true } return false } func (worker *AccountHandler) saveErrorState(infos map[string]string, err string) error { // ensure errors data fields are present if _, ok := infos[lastErrorKey]; !ok { infos[lastErrorKey] = "" } if _, ok := infos[dateFirstErrorKey]; !ok { infos[dateFirstErrorKey] = "" } if _, ok := infos[dateLastErrorKey]; !ok { infos[dateLastErrorKey] = "" } if _, ok := infos[errorsCountKey]; !ok { infos[errorsCountKey] = "0" } // log last error infos[lastErrorKey] = "Mastodon connection failed : " + err log.Warnf("Mastodon connection failed for remote identity %s : %s", worker.userAccount.remoteID, err) // increment counter count, _ := strconv.Atoi(infos[errorsCountKey]) count++ infos[errorsCountKey] = strconv.Itoa(count) // update dates lastDate := time.Now() var firstDate time.Time firstDate, _ = time.Parse(time.RFC3339, infos[dateFirstErrorKey]) if firstDate.IsZero() { firstDate = lastDate } infos[dateFirstErrorKey] = firstDate.Format(time.RFC3339) infos[dateLastErrorKey] = lastDate.Format(time.RFC3339) // check failuresThreshold if lastDate.Sub(firstDate)/time.Hour > failuresThreshold { // disable remote identity err := worker.broker.Store.UpdateUserIdentity(&UserIdentity{ UserId: worker.userAccount.userID, Id: worker.userAccount.remoteID, }, map[string]interface{}{ "Status": "inactive", }) if err != nil { log.WithError(err).Warnf("[saveErrorState] failed to deactivate remote identity %s for user %s", worker.userAccount.remoteID, worker.userAccount.userID) } // send nats message to idpoller to stop polling order := RemoteIDNatsMessage{ IdentityId: worker.userAccount.remoteID.String(), Order: "delete", Protocol: "mastodon", UserId: worker.userAccount.userID.String(), } jorder, jerr := json.Marshal(order) if jerr == nil { e := worker.broker.NatsConn.Publish(worker.broker.Config.NatsTopicPollerCache, jorder) if e != nil { log.WithError(e).Warnf("[saveErrorState] failed to publish delete order to idpoller") } } } // update UserIdentity in db return worker.broker.Store.UpdateRemoteInfosMap(worker.userAccount.userID.String(), worker.userAccount.remoteID.String(), infos) } // ByAscID implements sort interface type ByAscID []mastodon.Status func (bri ByAscID) Len() int { return len(bri) } func (bri ByAscID) Less(i, j int) bool { return bri[i].ID < bri[j].ID } func (bri ByAscID) Swap(i, j int) { bri[i], bri[j] = bri[j], bri[i] } ================================================ FILE: src/backend/protocols/go.mastodon/cmd/mastodonworker/cli_cmds/root.go ================================================ // Copyleft (ɔ) 2018 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package cmd import ( log "github.com/Sirupsen/logrus" "github.com/spf13/cobra" ) var ( verbose bool version bool RootCmd = &cobra.Command{ Use: "mastodond", Short: "Mastodon API daemon", Long: `mastodond is a daemon that connect to Mastodon accounts on one side and to our NATS queues on other side to executes IO operations with Mastodon API`, Run: nil, } ) const __version__ = "0.23.0" func init() { cobra.OnInitialize() RootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "print out more debug information") RootCmd.PersistentFlags().BoolVarP(&version, "version", "V", false, "print out the version of this program") RootCmd.Run = func(cmd *cobra.Command, args []string) { if version { log.Infof("mastodond version %s", __version__) } if len(args) == 0 { cmd.Help() } } RootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { if verbose { log.SetLevel(log.DebugLevel) } else { log.SetLevel(log.InfoLevel) } } RootCmd.AddCommand(versionCmd) } var versionCmd = &cobra.Command{ Use: "version", Short: "Print the version number of mastodond", Long: `All software has versions. This is mastodond's`, Run: func(cmd *cobra.Command, args []string) { log.Infof("mastodond version %s", __version__) }, } ================================================ FILE: src/backend/protocols/go.mastodon/cmd/mastodonworker/cli_cmds/start.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package cmd import ( "crypto/rand" "fmt" mast "github.com/CaliOpen/Caliopen/src/backend/protocols/go.mastodon" log "github.com/Sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" "io" "os" "os/signal" "sync" "syscall" "time" ) const ( shutdownTimeout = 3 // minutes to wait before forcing shutdown ) var ( configPath string configFile string pidFile string signalChannel chan os.Signal mastodonWorkers []*mast.Worker startCmd = &cobra.Command{ Use: "start", Short: "Starts a pool of mastodon API worker(s)", Run: start, } ) func init() { startCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "mastodonworker", "Name of the configuration file, without extension. (YAML, TOML, JSON… allowed)") startCmd.PersistentFlags().StringVarP(&configPath, "configpath", "", "../../../../configs/", "Main config file path.") startCmd.PersistentFlags().StringVarP(&pidFile, "pid-file", "p", "/var/run/caliopen_mastodond.pid", "Path to the pid file") RootCmd.AddCommand(startCmd) signalChannel = make(chan os.Signal, 1) } func start(cmd *cobra.Command, args []string) { var conf mast.WorkerConfig err := readConfig(&conf) if err != nil { log.WithError(err).Fatal("Error while reading config") } // Write out our PID if len(pidFile) > 0 { if f, err := os.Create(pidFile); err == nil { defer f.Close() if _, err := f.WriteString(fmt.Sprintf("%d", os.Getpid())); err == nil { f.Sync() } else { log.WithError(err).Warnf("Error while writing pidFile (%s)", pidFile) } } else { log.WithError(err).Warnf("Error while creating pidFile (%s)", pidFile) } } // init and start worker(s) var i uint8 mastodonWorkers = make([]*mast.Worker, conf.Workers) for i = 0; i < conf.Workers; i++ { log.Infof("Initializing mastodon worker %d", i) mastodonWorkers[i], err = mast.InitWorker(conf, verbose, randomIdentifier()) if err != nil { log.WithError(err).Fatal("failed to init worker") } go mastodonWorkers[i].Start() } // listening mode, waiting for nats orders to add/update workers or os sig to shutdown sigHandler(mastodonWorkers) } // ReadConfig which should be called at startup, or when a SIG_HUP is caught func readConfig(config *mast.WorkerConfig) error { // load in the main config. Reading from YAML, TOML, JSON, HCL and Java properties config files v := viper.New() v.SetConfigName(configFile) // name of config file (without extension) v.AddConfigPath(configPath) // path to look for the config file in v.AddConfigPath("$CALIOPENROOT/src/backend/configs/") // call multiple times to add many search paths v.AddConfigPath(".") // optionally look for config in the working directory err := v.ReadInConfig() // Find and read the config file*/ if err != nil { log.WithError(err).Infof("Could not read main config file <%s>.", configFile) return err } err = v.Unmarshal(config) if err != nil { log.WithError(err).Infof("Could not parse config file: <%s>", configFile) return err } return nil } func sigHandler(workers []*mast.Worker) { signal.Notify(signalChannel, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGINT, syscall.SIGKILL) for sig := range signalChannel { if sig == syscall.SIGHUP { // TODO: handle SIGHUP } else if sig == syscall.SIGTERM || sig == syscall.SIGQUIT || sig == syscall.SIGINT || sig == syscall.SIGKILL { log.Infof("Shutdown signal caught. Gracefully halting %d workers within 3 minutes timeframe…", len(workers)) wg := new(sync.WaitGroup) wg.Add(len(workers)) for i := range workers { workers[i].HaltGroup = wg } // timeout mechanism to avoid infinite wait c := make(chan struct{}) go func() { defer close(c) wg.Wait() }() select { case <-c: log.Info("Shutdown completed, exiting") os.Exit(0) case <-time.After(shutdownTimeout * time.Minute): log.Warn("Shutdown timeout, force exiting") os.Exit(0) } } else { os.Exit(0) } } } func randomIdentifier() string { var buf [4]byte _, err := io.ReadFull(rand.Reader, buf[:]) if err != nil { return "00000000" } return fmt.Sprintf("%x", buf[:]) } ================================================ FILE: src/backend/protocols/go.mastodon/cmd/mastodonworker/main.go ================================================ // Copyleft (ɔ) 2018 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package main import ( "fmt" "github.com/CaliOpen/Caliopen/src/backend/protocols/go.mastodon/cmd/mastodonworker/cli_cmds" "os" ) func main() { if err := cmd.RootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(-1) } } ================================================ FILE: src/backend/protocols/go.mastodon/messaging.go ================================================ // Copyleft (ɔ) 2018 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package mastodonworker import ( "encoding/json" "fmt" "github.com/CaliOpen/Caliopen/src/backend/brokers/go.mastodon" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "github.com/nats-io/go-nats" "github.com/pkg/errors" "time" ) // WorkerMsgHandler handles message coming from idpoller func (w *Worker) WorkerMsgHandler(msg *nats.Msg) { message := BrokerOrder{} err := json.Unmarshal(msg.Data, &message) if err != nil { log.WithError(err).Errorf("Unable to unmarshal message from NATS. Payload was <%s>", string(msg.Data)) return } switch message.Order { case noPendingJobErr: return case "sync": log.Infof("received sync order for remote mastodon ID %s", message.IdentityId) if accountWorker := w.getOrCreateHandler(message.UserId, message.IdentityId); accountWorker != nil { select { case accountWorker.AccountDesk <- PollDM: log.Infof("[DMmsgHandler] ordering to pollDM for remote %s (user %s)", message.IdentityId, message.UserId) case <-time.After(30 * time.Second): log.Warnf("[DMmsgHandler] worker's desk is full for remote %s (user %s)", message.IdentityId, message.UserId) } } else { log.Warnf("[DMmsgHandler] failed to get a worker for remote %s (user %s)", message.IdentityId, message.UserId) w.natsReplyError(msg, errors.New("[DMmsgHandler] failed to get a worker")) } case "reload_worker": log.Infof("received reload_worker order for remote mastodon ID %s", message.IdentityId) //TODO: order to force refreshing cache data for an account case "add_worker": log.Infof("received add_worker order for remote mastodon ID %s", message.IdentityId) accountWorker := w.getOrCreateHandler(message.UserId, message.IdentityId) if accountWorker == nil { log.WithError(err).Warnf("[WorkerMsgHandler] failed to create new worker for remote %s (user %s)", message.IdentityId, message.UserId) w.natsReplyError(msg, errors.New("[DMmsgHandler] failed to get a worker")) } case "remove_worker": log.Infof("received remove_worker order for remote mastodon ID %s", message.IdentityId) // TODO } } // DMmsgHandler handles messages coming on topic dedicated to DM management func (w *Worker) DMmsgHandler(msg *nats.Msg) { message := BrokerOrder{} err := json.Unmarshal(msg.Data, &message) if err != nil { log.WithError(err).Errorf("Unable to unmarshal message from NATS. Payload was <%s>", string(msg.Data)) return } switch message.Order { case "deliver": if accountWorker := w.getOrCreateHandler(message.UserId, message.IdentityId); accountWorker != nil { com := mastodonbroker.NatsCom{ Order: message, Ack: make(chan *DeliveryAck), } select { case accountWorker.broker.Connectors.Egress <- com: log.Infof("[DMmsgHandler] sending DM for remote %s (user %s)", message.IdentityId, message.UserId) // non-blocking wait for delivery ack go func(com mastodonbroker.NatsCom) { select { case resp := <-com.Ack: if resp.Err { w.natsReplyError(msg, errors.New(resp.Response)) } else { ack := DeliveryAck{ Err: false, Response: "OK", } json_resp, _ := json.Marshal(ack) w.NatsConn.Publish(msg.Reply, json_resp) } case <-time.After(30 * time.Second): w.natsReplyError(msg, errors.New("[DMmsgHandler] timeout waiting broker delivery ack")) } }(com) case <-time.After(30 * time.Second): log.Warnf("[DMmsgHandler] worker's Egress connectors is full for remote %s (user %s)", message.IdentityId, message.UserId) w.natsReplyError(msg, errors.New("[DMmsgHandler] failed to get a worker")) } } else { w.natsReplyError(msg, errors.New("[DMmsgHandler] failed to get a worker")) } default: w.natsReplyError(msg, errors.New("not implemented")) } } func (w *Worker) natsReplyError(msg *nats.Msg, err error) { log.WithError(err).Warnf("mastodon broker [outbound] : error when processing incoming nats message : %v", *msg) ack := DeliveryAck{ Err: true, Response: fmt.Sprintf("failed to send message with error « %s » ", err), //TODO } json_resp, _ := json.Marshal(ack) w.NatsConn.Publish(msg.Reply, json_resp) } ================================================ FILE: src/backend/protocols/go.mastodon/worker.go ================================================ package mastodonworker import ( "errors" "fmt" broker "github.com/CaliOpen/Caliopen/src/backend/brokers/go.mastodon" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/index/elasticsearch" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/cassandra" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/Notifications" log "github.com/Sirupsen/logrus" "github.com/gocql/gocql" "github.com/nats-io/go-nats" "sync" "time" ) type ( Worker struct { AccountHandlers map[string]*AccountHandler // one worker per active Mastodon account HaltGroup *sync.WaitGroup Index backends.LDAIndex Id string NatsConn *nats.Conn NatsSubs []*nats.Subscription Notifier *Notifications.Notifier Store backends.LDAStore WorkersGuard *sync.RWMutex Conf WorkerConfig Desk chan DeskMessage // chan to allow accountworkers to communicate with their master } WorkerConfig struct { Workers uint8 `mapstructure:"workers"` MastodonAppKey string `mapstructure:"mastodon_app_key"` MastodonAppSecret string `mapstructure:"mastodon_app_secret"` BrokerConfig broker.BrokerConfig `mapstructure:"BrokerConfig"` } DeskMessage struct { order string account *AccountHandler } ) const ( failuresThreshold = 72 // how many hours to wait before disabling a faulty remote. noPendingJobErr = "no pending job" pollThrottling = 10 * time.Second needJobOrderStr = `{"worker":"%s","order":{"order":"need_job"}}` closeAccountOrder = "close_account" ) func InitWorker(conf WorkerConfig, verboseLog bool, id string) (worker *Worker, err error) { if verboseLog { log.SetLevel(log.DebugLevel) } worker = &Worker{ AccountHandlers: map[string]*AccountHandler{}, Conf: conf, Id: id, WorkersGuard: new(sync.RWMutex), } // init Store switch conf.BrokerConfig.StoreName { case "cassandra": c := store.CassandraConfig{ Hosts: conf.BrokerConfig.StoreConfig.Hosts, Keyspace: conf.BrokerConfig.StoreConfig.Keyspace, Consistency: gocql.Consistency(conf.BrokerConfig.StoreConfig.Consistency), SizeLimit: conf.BrokerConfig.StoreConfig.SizeLimit, UseVault: conf.BrokerConfig.StoreConfig.UseVault, } if conf.BrokerConfig.StoreConfig.ObjectStore == "s3" { c.WithObjStore = true c.Endpoint = conf.BrokerConfig.StoreConfig.OSSConfig.Endpoint c.AccessKey = conf.BrokerConfig.StoreConfig.OSSConfig.AccessKey c.SecretKey = conf.BrokerConfig.StoreConfig.OSSConfig.SecretKey c.RawMsgBucket = conf.BrokerConfig.StoreConfig.OSSConfig.Buckets["raw_messages"] c.AttachmentBucket = conf.BrokerConfig.StoreConfig.OSSConfig.Buckets["temporary_attachments"] c.Location = conf.BrokerConfig.StoreConfig.OSSConfig.Location } if conf.BrokerConfig.StoreConfig.UseVault { c.HVaultConfig.Url = conf.BrokerConfig.StoreConfig.VaultConfig.Url c.HVaultConfig.Username = conf.BrokerConfig.StoreConfig.VaultConfig.Username c.HVaultConfig.Password = conf.BrokerConfig.StoreConfig.VaultConfig.Password } b, e := store.InitializeCassandraBackend(c) if e != nil { err = e log.WithError(err).Warnf("[MastodonWorker] initialization of %s backend failed", conf.BrokerConfig.StoreName) return } worker.Store = backends.LDAStore(b) // type conversion to LDA interface default: log.Warnf("[MastodonWorker] unknown store backend: %s", conf.BrokerConfig.StoreName) err = errors.New("[MastodonWorker] unknown store backend") return } // init Index switch conf.BrokerConfig.LDAConfig.IndexName { case "elasticsearch": c := index.ElasticSearchConfig{ Urls: conf.BrokerConfig.LDAConfig.IndexConfig.Urls, } i, e := index.InitializeElasticSearchIndex(c) if e != nil { err = e log.WithError(err).Warnf("[MastodonBroker] initialization of %s backend failed", conf.BrokerConfig.IndexName) return } worker.Index = backends.LDAIndex(i) // type conversion to LDA interface default: log.Warnf("[MastodonBroker] unknown index backend: %s", conf.BrokerConfig.LDAConfig.IndexName) err = errors.New("[MastodonBroker] unknown index backend") return } worker.NatsConn, err = nats.Connect(conf.BrokerConfig.NatsURL) if err != nil { log.WithError(err).Warn("[MastodonBroker] initalization of NATS connexion failed") return } caliopenConfig := CaliopenConfig{ NotifierConfig: conf.BrokerConfig.LDAConfig.NotifierConfig, NatsConfig: NatsConfig{ Url: conf.BrokerConfig.NatsURL, }, RESTstoreConfig: RESTstoreConfig{ BackendName: conf.BrokerConfig.StoreName, Consistency: conf.BrokerConfig.StoreConfig.Consistency, Hosts: conf.BrokerConfig.StoreConfig.Hosts, Keyspace: conf.BrokerConfig.StoreConfig.Keyspace, OSSConfig: conf.BrokerConfig.StoreConfig.OSSConfig, ObjStoreType: conf.BrokerConfig.StoreConfig.ObjectStore, SizeLimit: conf.BrokerConfig.StoreConfig.SizeLimit, }, RESTindexConfig: RESTIndexConfig{ Hosts: conf.BrokerConfig.LDAConfig.IndexConfig.Urls, IndexName: conf.BrokerConfig.LDAConfig.IndexName, }, } worker.Notifier = Notifications.NewNotificationsFacility(caliopenConfig, worker.NatsConn) // init Nats connector worker.NatsConn, err = nats.Connect(conf.BrokerConfig.NatsURL) if err != nil { log.WithError(err).Fatal("[MastodonWorker] initialization of NATS connexion failed") } worker.NatsSubs = make([]*nats.Subscription, 1) worker.NatsSubs[0], err = worker.NatsConn.QueueSubscribe(conf.BrokerConfig.NatsTopicDMs, conf.BrokerConfig.NatsQueue, worker.DMmsgHandler) if err != nil { log.WithError(err).Fatal("[MastodonWorker] initialization of NATS fetcher subscription failed") } err = worker.NatsConn.Flush() if err != nil { log.WithError(err).Fatal("[MastodonWorker] initialization of NATS fetcher subscription failed") } worker.Desk = make(chan DeskMessage, 2) return worker, nil } func (worker *Worker) Start(throttling ...time.Duration) { var throttle time.Duration if len(throttling) == 1 && throttling[0] != 0 { throttle = throttling[0] } else { throttle = pollThrottling } go func() { for msg := range worker.Desk { switch msg.order { case closeAccountOrder: if msg.account != nil { worker.RemoveAccountHandler(msg.account) } default: log.Debugf("[MastodonWorker] received unknown order « %s » from account %s", msg.order, msg.account.userAccount.userID.String()+msg.account.userAccount.remoteID.String()) } } }() // start throttled jobs polling log.Infof("Mastodon worker %s starting with %d sec throttling", worker.Id, throttle/time.Second) for { start := time.Now() requestOrder := []byte(fmt.Sprintf(needJobOrderStr, worker.Id)) log.Infof("Mastodon worker %s is requesting jobs to idpoller", worker.Id) resp, err := worker.NatsConn.Request(worker.Conf.BrokerConfig.NatsTopicPoller, requestOrder, time.Minute) if err != nil { log.WithError(err).Warnf("[worker %s] failed to request pending jobs on nats", worker.Id) } else { worker.WorkerMsgHandler(resp) } // check for interrupt after job is finished if worker.HaltGroup != nil { worker.stop() break } elapsed := time.Now().Sub(start) if elapsed < throttle { time.Sleep(throttle - elapsed) } } } func (worker *Worker) stop() { for _, w := range worker.AccountHandlers { w.AccountDesk <- Stop } for _, sub := range worker.NatsSubs { sub.Unsubscribe() } worker.NatsConn.Close() worker.Store.Close() worker.Index.Close() worker.HaltGroup.Done() close(worker.Desk) log.Infof("worker %s stopped", worker.Id) } // getOrCreateHandler returns a pointer to a worker already in cache // or tries to create a new worker for the remote identity if not. // returns nil if get or create failed. func (w *Worker) getOrCreateHandler(userId, remoteId string) *AccountHandler { w.WorkersGuard.RLock() if accountHandler, ok := w.AccountHandlers[userId+remoteId]; ok { w.WorkersGuard.RUnlock() return accountHandler } else { w.WorkersGuard.RUnlock() log.Infof("[getOrCreateHandler] failed to retrieve registered worker for remote %s (user %s). Trying to add one.", remoteId, userId) if userId == "" || remoteId == "" { return nil } accountHandler, err := NewAccountHandler(userId, remoteId, *w) if err != nil { log.WithError(err).Warnf("[getOrCreateHandler] failed to create new worker for remote %s (user %s)", remoteId, userId) return nil } w.RegisterAccountHandler(accountHandler) go accountHandler.Start() return accountHandler } } func (w *Worker) RegisterAccountHandler(accountHandler *AccountHandler) { workerKey := accountHandler.userAccount.userID.String() + accountHandler.userAccount.remoteID.String() // stop & remove handler first if it's already registered w.WorkersGuard.RLock() registeredHandler, ok := w.AccountHandlers[workerKey] w.WorkersGuard.RUnlock() if ok { w.RemoveAccountHandler(registeredHandler) } w.WorkersGuard.Lock() w.AccountHandlers[workerKey] = accountHandler w.WorkersGuard.Unlock() } func (w *Worker) RemoveAccountHandler(accountHandler *AccountHandler) { workerKey := accountHandler.userAccount.userID.String() + accountHandler.userAccount.remoteID.String() w.WorkersGuard.Lock() accountHandler.Stop() delete(w.AccountHandlers, workerKey) w.WorkersGuard.Unlock() } ================================================ FILE: src/backend/protocols/go.smtp/cmd/caliopen_lmtpd/cli_cmds/root.go ================================================ package cmd import ( log "github.com/Sirupsen/logrus" "github.com/spf13/cobra" ) var ( verbose bool version bool RootCmd = &cobra.Command{ Use: "caliopen_lmtpd", Short: "LMTP daemon", Long: `LMTP daemon for the purpose of bridging MTAs to our local delivery agent.`, Run: nil, } ) const __version__ = "0.23.0" func init() { cobra.OnInitialize() RootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "print out more debug information") RootCmd.PersistentFlags().BoolVarP(&version, "version", "V", false, "print out the version of this program") RootCmd.Run = func(cmd *cobra.Command, args []string) { if version { log.Infof("Caliopen SMTPd version %s", __version__) } } RootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { if verbose { log.SetLevel(log.DebugLevel) } else { log.SetLevel(log.InfoLevel) } } RootCmd.AddCommand(versionCmd) } var versionCmd = &cobra.Command{ Use: "version", Short: "Print the version number of Caliopen SMTPd", Long: `All software has versions. This is Caliopen SMTPd's`, Run: func(cmd *cobra.Command, args []string) { log.Infof("Caliopen SMTPd version %s", __version__) }, } ================================================ FILE: src/backend/protocols/go.smtp/cmd/caliopen_lmtpd/cli_cmds/serve.go ================================================ package cmd import ( "errors" "fmt" csmtp "github.com/CaliOpen/Caliopen/src/backend/protocols/go.smtp" log "github.com/Sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" "os" "os/signal" "syscall" ) var ( configPath string configFile string pidFile string signalChannel chan os.Signal // for trapping SIG_HUP cmdConfig CmdConfig serveCmd = &cobra.Command{ Use: "serve", Short: "Start the caliopen LMTP server", Run: serve, } ) func init() { serveCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "lmtp", "Name of the configuration file, without extension. (YAML, TOML, JSON… allowed)") serveCmd.PersistentFlags().StringVarP(&configPath, "configpath", "", "../../../../configs/", "Main config file path.") serveCmd.PersistentFlags().StringVarP(&pidFile, "pid-file", "p", "/var/run/caliopen_lmtpd.pid", "Path to the pid file") RootCmd.AddCommand(serveCmd) signalChannel = make(chan os.Signal, 1) cmdConfig = CmdConfig{} } func sigHandler() { // handle SIGHUP for reloading the configuration while running signal.Notify(signalChannel, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGINT, syscall.SIGKILL) for sig := range signalChannel { if sig == syscall.SIGHUP { err := readConfig(&cmdConfig) if err != nil { log.WithError(err).Error("Error while ReadConfig (reload)") } else { log.Info("Configuration is reloaded") } // TODO: reinitialize } else if sig == syscall.SIGTERM || sig == syscall.SIGQUIT || sig == syscall.SIGINT { log.Infof("Shutdown signal caught") csmtp.ShutdownServer() log.Infof("Shutdown completed, exiting.") os.Exit(0) } else { os.Exit(0) } } } func serve(cmd *cobra.Command, args []string) { err := readConfig(&cmdConfig) if err != nil { log.WithError(err).Fatal("Error while reading config") } // Write out our PID if len(pidFile) > 0 { if f, err := os.Create(pidFile); err == nil { defer f.Close() if _, err := f.WriteString(fmt.Sprintf("%d", os.Getpid())); err == nil { f.Sync() } else { log.WithError(err).Warnf("Error while writing pidFile (%s)", pidFile) } } else { log.WithError(err).Warnf("Error while creating pidFile (%s)", pidFile) } } err = csmtp.InitializeServer(csmtp.SMTPConfig(cmdConfig)) if err != nil { log.WithError(err).Fatal("Failed to init LMTP server") } go csmtp.StartServer() sigHandler() } type CmdConfig csmtp.SMTPConfig // ReadConfig which should be called at startup, or when a SIG_HUP is caught func readConfig(config *CmdConfig) error { // load in the main config. Reading from YAML, TOML, JSON, HCL and Java properties config files v := viper.New() v.SetConfigName(configFile) // name of config file (without extension) v.AddConfigPath(configPath) // path to look for the config file in v.AddConfigPath("$CALIOPENROOT/src/backend/configs/") // call multiple times to add many search paths v.AddConfigPath(".") // optionally look for config in the working directory err := v.ReadInConfig() // Find and read the config file*/ if err != nil { log.WithError(err).Infof("Could not read main config file <%s>.", configFile) return err } err = v.Unmarshal(config) if err != nil { log.WithError(err).Infof("Could not parse config file: <%s>", configFile) return err } if len(config.AppConfig.AllowedHosts) == 0 { return errors.New("Empty `allowed_hosts` is not allowed") } config.AppConfig.AppVersion = __version__ config.LDAConfig.AppVersion = config.AppConfig.AppVersion config.LDAConfig.PrimaryMailHost = config.AppConfig.PrimaryMailHost return nil } ================================================ FILE: src/backend/protocols/go.smtp/cmd/caliopen_lmtpd/main.go ================================================ package main import ( "fmt" "github.com/CaliOpen/Caliopen/src/backend/protocols/go.smtp/cmd/caliopen_lmtpd/cli_cmds" "os" ) func main() { if err := cmd.RootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(-1) } } ================================================ FILE: src/backend/protocols/go.smtp/config.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package caliopen_smtp import ( . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" ) type ( SMTPConfig struct { AppConfig AppConfig LDAConfig LDAConfig } AppConfig struct { AppVersion string `mapstructure:"version"` Servers []ServerConfig `mapstructure:"inbound_servers"` AllowedHosts []string `mapstructure:"allowed_hosts"` PrimaryMailHost string `mapstructure:"primary_mail_host"` SubmitAddress string `mapstructure:"submit_address"` SubmitPort int `mapstructure:"submit_port"` SubmitUser string `mapstructure:"submit_user"` SubmitPassword string `mapstructure:"submit_password"` OutWorkers int `mapstructure:"submit_workers"` } // ServerConfig specifies config options for a single smtp server ServerConfig struct { IsEnabled bool `mapstructure:"is_enabled"` Hostname string `mapstructure:"host_name"` AllowedHosts []string MaxSize uint64 `mapstructure:"max_size"` //max size for emails PrivateKeyFile string `mapstructure:"private_key_file"` PublicKeyFile string `mapstructure:"public_key_file"` Timeout int `mapstructure:"timeout"` ListenInterface string `mapstructure:"listen_interface"` StartTLSOn bool `mapstructure:"start_tls_on,omitempty"` TLSAlwaysOn bool `mapstructure:"tls_always_on,omitempty"` MaxClients int `mapstructure:"max_clients"` } ) ================================================ FILE: src/backend/protocols/go.smtp/envelope.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package caliopen_smtp import ( "crypto/tls" "fmt" "strings" "time" ) // Envelope holds a message type SmtpEnvelope struct { Sender string Recipients []string Data []byte } // AddReceivedLine prepends a Received header to the Data func (env *SmtpEnvelope) AddReceivedLine(peer Peer) { tlsDetails := "" tlsVersions := map[uint16]string{ tls.VersionSSL30: "SSL3.0", tls.VersionTLS10: "TLS1.0", tls.VersionTLS11: "TLS1.1", tls.VersionTLS12: "TLS1.2", } if peer.TLS != nil { tlsDetails = fmt.Sprintf( "\r\n\t(version=%s cipher=0x%x);", tlsVersions[peer.TLS.Version], peer.TLS.CipherSuite, ) } line := wrap([]byte(fmt.Sprintf( "Received: from %s [%s] by %s with %s;%s\r\n\t%s\r\n", peer.HeloName, strings.Split(peer.Addr.String(), ":")[0], peer.ServerName, peer.Protocol, tlsDetails, time.Now().Format("Mon Jan 2 15:04:05 -0700 2006"), ))) env.Data = append(env.Data, line...) // Move the new Received line up front copy(env.Data[len(line):], env.Data[0:len(env.Data)-len(line)]) copy(env.Data, line) } // Wrap a byte slice paragraph for use in SMTP header func wrap(sl []byte) []byte { length := 0 for i := 0; i < len(sl); i++ { if length > 76 && sl[i] == ' ' { sl = append(sl, 0, 0) copy(sl[i+2:], sl[i:]) sl[i] = '\r' sl[i+1] = '\n' sl[i+2] = '\t' i += 2 length = 0 } if sl[i] == '\n' { length = 0 } length++ } return sl } ================================================ FILE: src/backend/protocols/go.smtp/lda.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. // // SMTP server to handle in/out emails from/to MTAs package caliopen_smtp import ( broker "github.com/CaliOpen/Caliopen/src/backend/brokers/go.emails" log "github.com/Sirupsen/logrus" "os/exec" "strconv" "strings" ) //Local Delivery Agent, in charge of IO between SMTP server and our email broker type Lda struct { Config SMTPConfig broker *broker.EmailBroker brokerConnectors broker.EmailBrokerConnectors inboundListener *Server outboundListener *submitter } func (lda *Lda) initialize(config SMTPConfig) (err error) { lda.Config = config lda.broker, lda.brokerConnectors, err = broker.Initialize(config.LDAConfig) return err } func (lda *Lda) start() (err error) { // Check that max clients is not greater than system open file limit. fileLimit := getFileLimit() if fileLimit > 0 { maxClients := 0 for _, s := range lda.Config.AppConfig.Servers { maxClients += s.MaxClients } if maxClients > fileLimit { log.Fatalf("Combined max clients for all servers (%d) is greater than open file limit (%d). "+ "Please increase your open file limit or decrease max clients.", maxClients, fileLimit) } } // launch outbound chan listener lda.outboundListener, err = lda.newSubmitter() if err != nil { log.Warn("LDA submitter initialization failed") } go func() { lda.runSubmitterAgent() }() return } func (lda *Lda) shutdown() error { lda.broker.ShutDown() return nil } func getFileLimit() int { cmd := exec.Command("ulimit", "-n") out, err := cmd.Output() if err != nil { return -1 } limit, err := strconv.Atoi(strings.TrimSpace(string(out))) if err != nil { return -1 } return limit } ================================================ FILE: src/backend/protocols/go.smtp/lmtpd.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package caliopen_smtp import ( "bufio" "crypto/tls" "errors" "fmt" "log" "net" "time" ) // Server defines the parameters for running the SMTP server type Server struct { ListenAddr string //(default: "localhost:2525") Hostname string // Server hostname. (default: "localhost.localdomain") WelcomeMessage string // Initial server banner. (default: " ESMTP ready.") ReadTimeout time.Duration // Socket timeout for read operations. (default: 60s) WriteTimeout time.Duration // Socket timeout for write operations. (default: 60s) DataTimeout time.Duration // Socket timeout for DATA command (default: 5m) MaxConnections int // Max concurrent connections, use -1 to disable. (default: 100) MaxMessageSize int // Max message size in bytes. (default: 10240000) MaxRecipients int // Max RCPT TO calls for each envelope. (default: 100) // New e-mails are handed off to this function. // Can be left empty for a NOOP server. // If an error is returned, it will be reported in the SMTP session. Handler func(peer Peer, env SmtpEnvelope) error // Enable various checks during the SMTP session. // Can be left empty for no restrictions. // If an error is returned, it will be reported in the SMTP session. // Use the Error struct for access to error codes. ConnectionChecker func(peer Peer) error // Called upon new connection. HeloChecker func(peer Peer, name string) error // Called after HELO/EHLO. SenderChecker func(peer Peer, addr string) error // Called after MAIL FROM. RecipientChecker func(peer Peer, addr string) error // Called after each RCPT TO. // Enable PLAIN/LOGIN authentication, only available after STARTTLS. // Can be left empty for no authentication support. Authenticator func(peer Peer, username, password string) error EnableXCLIENT bool // Enable XCLIENT support (default: false) TLSConfig *tls.Config // Enable STARTTLS support. ForceTLS bool // Force STARTTLS usage. ProtocolLogger *log.Logger } // Protocol represents the protocol used in the SMTP session type Protocol string const ( SMTP Protocol = "SMTP" ESMTP = "ESMTP" ) // Peer represents the client connecting to the server type Peer struct { HeloName string // Server name used in HELO/EHLO command Username string // Username from authentication, if authenticated Password string // Password from authentication, if authenticated Protocol Protocol // Protocol used, SMTP or ESMTP ServerName string // A copy of Server.Hostname Addr net.Addr // Network address TLS *tls.ConnectionState // TLS Connection details, if on TLS } // Error represents an Error reported in the SMTP session. type Error struct { Code int // The integer error code Message string // The error message } // Error returns a string representation of the SMTP error func (e Error) Error() string { return fmt.Sprintf("%d %s", e.Code, e.Message) } type session struct { server *Server peer Peer envelope *SmtpEnvelope conn net.Conn reader *bufio.Reader writer *bufio.Writer scanner *bufio.Scanner tls bool } func (srv *Server) newSession(c net.Conn) (s *session) { s = &session{ server: srv, conn: c, reader: bufio.NewReader(c), writer: bufio.NewWriter(c), peer: Peer{ Addr: c.RemoteAddr(), ServerName: srv.Hostname, }, } s.scanner = bufio.NewScanner(s.reader) return } // Feed Server struct with config func (srv *Server) initialize(conf SMTPConfig) error { if lda == nil { return errors.New("unable to init smtpd : LDA is nil") } //TODO: handle multiple servers srv.ListenAddr = conf.AppConfig.Servers[0].ListenInterface srv.Hostname = conf.AppConfig.Servers[0].Hostname srv.ForceTLS = conf.AppConfig.Servers[0].TLSAlwaysOn srv.MaxConnections = conf.AppConfig.Servers[0].MaxClients srv.MaxMessageSize = int(conf.AppConfig.Servers[0].MaxSize) srv.Handler = lda.handler return nil } func (srv *Server) start() (err error) { go srv.ListenAndServe() return } func (srv *Server) shutdown() (err error) { return nil } // ListenAndServe starts the SMTP server and listens on the address provided func (srv *Server) ListenAndServe() error { srv.configureDefaults() l, err := net.Listen("tcp", srv.ListenAddr) if err != nil { return err } return srv.Serve(l) } // Serve starts the SMTP server and listens on the Listener provided func (srv *Server) Serve(l net.Listener) error { srv.configureDefaults() defer l.Close() var limiter chan struct{} if srv.MaxConnections > 0 { limiter = make(chan struct{}, srv.MaxConnections) } else { limiter = nil } for { conn, e := l.Accept() if e != nil { if ne, ok := e.(net.Error); ok && ne.Temporary() { time.Sleep(time.Second) continue } return e } session := srv.newSession(conn) if limiter != nil { go func() { select { case limiter <- struct{}{}: session.serve() <-limiter default: session.reject() } }() } else { go session.serve() } } } func (srv *Server) configureDefaults() { if srv.MaxMessageSize == 0 { srv.MaxMessageSize = 10240000 } if srv.MaxConnections == 0 { srv.MaxConnections = 100 } if srv.MaxRecipients == 0 { srv.MaxRecipients = 100 } if srv.ReadTimeout == 0 { srv.ReadTimeout = time.Second * 60 } if srv.WriteTimeout == 0 { srv.WriteTimeout = time.Second * 60 } if srv.DataTimeout == 0 { srv.DataTimeout = time.Minute * 5 } if srv.ForceTLS && srv.TLSConfig == nil { log.Fatal("Cannot use ForceTLS with no TLSConfig") } if srv.Hostname == "" { srv.Hostname = "localhost.localdomain" } if srv.WelcomeMessage == "" { srv.WelcomeMessage = fmt.Sprintf("%s ESMTP ready.", srv.Hostname) } } func (session *session) serve() { defer session.close() session.welcome() for { for session.scanner.Scan() { session.handle(session.scanner.Text()) } err := session.scanner.Err() if err == bufio.ErrTooLong { session.reply(500, "Line too long") // Advance reader to the next newline session.reader.ReadString('\n') session.scanner = bufio.NewScanner(session.reader) // Reset and have the client start over. session.reset() continue } break } } func (session *session) reject() { session.reply(421, "Too busy. Try again later.") session.close() } func (session *session) reset() { session.envelope = nil } func (session *session) welcome() { if session.server.ConnectionChecker != nil { err := session.server.ConnectionChecker(session.peer) if err != nil { session.error(err) session.close() return } } session.reply(220, session.server.WelcomeMessage) } func (session *session) reply(code int, message string) { if session.server.ProtocolLogger != nil { session.server.ProtocolLogger.Printf("%s > %d %s", session.conn.RemoteAddr(), code, message) } fmt.Fprintf(session.writer, "%d %s\r\n", code, message) session.flush() } func (session *session) flush() { session.conn.SetWriteDeadline(time.Now().Add(session.server.WriteTimeout)) session.writer.Flush() session.conn.SetReadDeadline(time.Now().Add(session.server.ReadTimeout)) } func (session *session) error(err error) { if smtpdError, ok := err.(Error); ok { session.reply(smtpdError.Code, smtpdError.Message) } else { session.reply(502, fmt.Sprintf("%s", err)) } } func (session *session) extensions() []string { extensions := []string{ fmt.Sprintf("SIZE %d", session.server.MaxMessageSize), "8BITMIME", "PIPELINING", } if session.server.EnableXCLIENT { extensions = append(extensions, "XCLIENT") } if session.server.TLSConfig != nil && !session.tls { extensions = append(extensions, "STARTTLS") } if session.server.Authenticator != nil && session.tls { extensions = append(extensions, "AUTH PLAIN LOGIN") } return extensions } func (session *session) deliver() error { if session.server.Handler != nil { return session.server.Handler(session.peer, *session.envelope) } return nil } func (session *session) close() { session.writer.Flush() time.Sleep(200 * time.Millisecond) session.conn.Close() } ================================================ FILE: src/backend/protocols/go.smtp/oauth.go ================================================ package caliopen_smtp import ( "encoding/json" "fmt" "net/smtp" ) // The XOAUTH2 mechanism name. const Xoauth2 = "XOAUTH2" // An XOAUTH2 error. type Xoauth2Response struct { Status string `json:"status"` Schemes string `json:"schemes"` Scope string `json:"scope"` } // Implements error. func (err *Xoauth2Response) Error() string { return fmt.Sprintf("XOAUTH2 authentication error (%v)", err.Status) } // xoauth2Client is the smtp.Auth interface that implements the XOauth2 authentication mechanism. type Xoauth2Client struct { Username string Token string } func (a *Xoauth2Client) Start(server *smtp.ServerInfo) (string, []byte, error) { return Xoauth2, []byte("user=" + a.Username + "\x01auth=Bearer " + a.Token + "\x01\x01"), nil } func (a *Xoauth2Client) Next(fromServer []byte, more bool) ([]byte, error) { if more { xoauth2Err := &Xoauth2Response{} if err := json.Unmarshal(fromServer, xoauth2Err); err != nil { return nil, err } else { return nil, xoauth2Err } } return nil, nil } ================================================ FILE: src/backend/protocols/go.smtp/protocol.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package caliopen_smtp import ( "bufio" "bytes" "crypto/tls" "encoding/base64" "fmt" "io" "io/ioutil" "net" "net/textproto" "strconv" "strings" "time" ) type command struct { line string action string fields []string params []string } func parseLine(line string) (cmd command) { cmd.line = line cmd.fields = strings.Fields(line) if len(cmd.fields) > 0 { cmd.action = strings.ToUpper(cmd.fields[0]) if len(cmd.fields) > 1 { cmd.params = strings.Split(cmd.fields[1], ":") } } return } func (session *session) handle(line string) { if session.server.ProtocolLogger != nil { session.server.ProtocolLogger.Printf("%s < %s", session.conn.RemoteAddr(), line) } cmd := parseLine(line) // Commands are dispatched to the appropriate handler functions. // If a network error occurs during handling, the handler should // just return and let the error be handled on the next read. switch cmd.action { case "HELO": session.handleHELO(cmd) return case "EHLO": session.handleEHLO(cmd) return case "MAIL": session.handleMAIL(cmd) return case "RCPT": session.handleRCPT(cmd) return case "STARTTLS": session.handleSTARTTLS(cmd) return case "DATA": session.handleDATA(cmd) return case "RSET": session.handleRSET(cmd) return case "NOOP": session.handleNOOP(cmd) return case "QUIT": session.handleQUIT(cmd) return case "AUTH": session.handleAUTH(cmd) return case "XCLIENT": session.handleXCLIENT(cmd) return } session.reply(502, "Unsupported command.") } func (session *session) handleHELO(cmd command) { if len(cmd.fields) < 2 { session.reply(502, "Missing parameter") return } if session.peer.HeloName != "" { // Reset envelope in case of duplicate HELO session.reset() } if session.server.HeloChecker != nil { err := session.server.HeloChecker(session.peer, cmd.fields[1]) if err != nil { session.error(err) return } } session.peer.HeloName = cmd.fields[1] session.peer.Protocol = SMTP session.reply(250, "Go ahead") return } func (session *session) handleEHLO(cmd command) { if len(cmd.fields) < 2 { session.reply(502, "Missing parameter") return } if session.peer.HeloName != "" { // Reset envelope in case of duplicate EHLO session.reset() } if session.server.HeloChecker != nil { err := session.server.HeloChecker(session.peer, cmd.fields[1]) if err != nil { session.error(err) return } } session.peer.HeloName = cmd.fields[1] session.peer.Protocol = ESMTP fmt.Fprintf(session.writer, "250-%s\r\n", session.server.Hostname) extensions := session.extensions() if len(extensions) > 1 { for _, ext := range extensions[:len(extensions)-1] { fmt.Fprintf(session.writer, "250-%s\r\n", ext) } } session.reply(250, extensions[len(extensions)-1]) return } func (session *session) handleMAIL(cmd command) { if len(cmd.params) != 2 || strings.ToUpper(cmd.params[0]) != "FROM" { session.reply(502, "Invalid syntax.") return } if session.peer.HeloName == "" { session.reply(502, "Please introduce yourself first.") return } if !session.tls && session.server.ForceTLS { session.reply(502, "Please turn on TLS by issuing a STARTTLS command.") return } if session.envelope != nil { session.reply(502, "Duplicate MAIL") return } addr, err := parseAddress(cmd.params[1]) if err != nil { session.reply(502, "Ill-formatted e-mail address") return } if session.server.SenderChecker != nil { err = session.server.SenderChecker(session.peer, addr) if err != nil { session.error(err) return } } session.envelope = &SmtpEnvelope{ Sender: addr, } session.reply(250, "Go ahead") return } func (session *session) handleRCPT(cmd command) { if len(cmd.params) != 2 || strings.ToUpper(cmd.params[0]) != "TO" { session.reply(502, "Invalid syntax.") return } if session.envelope == nil { session.reply(502, "Missing MAIL FROM command.") return } if len(session.envelope.Recipients) >= session.server.MaxRecipients { session.reply(452, "Too many recipients") return } addr, err := parseAddress(cmd.params[1]) if err != nil { session.reply(502, "Ill-formatted e-mail address") return } if session.server.RecipientChecker != nil { err = session.server.RecipientChecker(session.peer, addr) if err != nil { session.error(err) return } } session.envelope.Recipients = append(session.envelope.Recipients, addr) session.reply(250, "Go ahead") return } func (session *session) handleSTARTTLS(cmd command) { if session.tls { session.reply(502, "Already running in TLS") return } if session.server.TLSConfig == nil { session.reply(502, "TLS not supported") return } tlsConn := tls.Server(session.conn, session.server.TLSConfig) session.reply(220, "Go ahead") if err := tlsConn.Handshake(); err != nil { session.reply(550, "Handshake error") return } // Reset envelope as a new EHLO/HELO is required after STARTTLS session.reset() // Reset deadlines on the underlying connection before I replace it // with a TLS connection session.conn.SetDeadline(time.Time{}) // Replace connection with a TLS connection session.conn = tlsConn session.reader = bufio.NewReader(tlsConn) session.writer = bufio.NewWriter(tlsConn) session.scanner = bufio.NewScanner(session.reader) session.tls = true // Save connection state on peer state := tlsConn.ConnectionState() session.peer.TLS = &state // Flush the connection to set new timeout deadlines session.flush() return } func (session *session) handleDATA(cmd command) { if session.envelope == nil || len(session.envelope.Recipients) == 0 { session.reply(502, "Missing RCPT TO command.") return } session.reply(354, "Go ahead. End your data with .") session.conn.SetDeadline(time.Now().Add(session.server.DataTimeout)) data := &bytes.Buffer{} reader := textproto.NewReader(session.reader).DotReader() _, err := io.CopyN(data, reader, int64(session.server.MaxMessageSize)) if err == io.EOF { // EOF was reached before MaxMessageSize // Accept and deliver message session.envelope.Data = data.Bytes() if err := session.deliver(); err != nil { session.error(err) } else { session.reply(250, "Thank you.") } session.reset() } if err != nil { // Network error, ignore return } // Discard the rest and report an error. _, err = io.Copy(ioutil.Discard, reader) if err != nil { // Network error, ignore return } session.reply(552, fmt.Sprintf( "Message exceeded max message size of %d bytes", session.server.MaxMessageSize, )) session.reset() return } func (session *session) handleRSET(cmd command) { session.reset() session.reply(250, "Go ahead") return } func (session *session) handleNOOP(cmd command) { session.reply(250, "Go ahead") return } func (session *session) handleQUIT(cmd command) { session.reply(221, "OK, bye") session.close() return } func (session *session) handleAUTH(cmd command) { if len(cmd.fields) < 2 { session.reply(502, "Invalid syntax.") return } if session.server.Authenticator == nil { session.reply(502, "AUTH not supported.") return } if session.peer.HeloName == "" { session.reply(502, "Please introduce yourself first.") return } if !session.tls { session.reply(502, "Cannot AUTH in plain text mode. Use STARTTLS.") return } mechanism := strings.ToUpper(cmd.fields[1]) username := "" password := "" switch mechanism { case "PLAIN": auth := "" if len(cmd.fields) < 3 { session.reply(334, "Give me your credentials") if !session.scanner.Scan() { return } auth = session.scanner.Text() } else { auth = cmd.fields[2] } data, err := base64.StdEncoding.DecodeString(auth) if err != nil { session.reply(502, "Couldn't decode your credentials") return } parts := bytes.Split(data, []byte{0}) if len(parts) != 3 { session.reply(502, "Couldn't decode your credentials") return } username = string(parts[1]) password = string(parts[2]) case "LOGIN": session.reply(334, "VXNlcm5hbWU6") if !session.scanner.Scan() { return } byteUsername, err := base64.StdEncoding.DecodeString(session.scanner.Text()) if err != nil { session.reply(502, "Couldn't decode your credentials") return } session.reply(334, "UGFzc3dvcmQ6") if !session.scanner.Scan() { return } bytePassword, err := base64.StdEncoding.DecodeString(session.scanner.Text()) if err != nil { session.reply(502, "Couldn't decode your credentials") return } username = string(byteUsername) password = string(bytePassword) default: session.reply(502, "Unknown authentication mechanism") return } err := session.server.Authenticator(session.peer, username, password) if err != nil { session.error(err) return } session.peer.Username = username session.peer.Password = password session.reply(235, "OK, you are now authenticated") } func (session *session) handleXCLIENT(cmd command) { if len(cmd.fields) < 2 { session.reply(502, "Invalid syntax.") return } if !session.server.EnableXCLIENT { session.reply(550, "XCLIENT not enabled") return } var ( newHeloName = "" newAddr net.IP = nil newTCPPort uint64 = 0 newUsername = "" newProto Protocol = "" ) for _, item := range cmd.fields[1:] { parts := strings.Split(item, "=") if len(parts) != 2 { session.reply(502, "Couldn't decode the command.") return } name := parts[0] value := parts[1] switch name { case "NAME": // Unused in smtpd package continue case "HELO": newHeloName = value continue case "ADDR": newAddr = net.ParseIP(value) continue case "PORT": var err error newTCPPort, err = strconv.ParseUint(value, 10, 16) if err != nil { session.reply(502, "Couldn't decode the command.") return } continue case "LOGIN": newUsername = value continue case "PROTO": if value == "SMTP" { newProto = SMTP } else if value == "ESMTP" { newProto = ESMTP } continue default: session.reply(502, "Couldn't decode the command.") return } } tcpAddr, ok := session.peer.Addr.(*net.TCPAddr) if !ok { session.reply(502, "Unsupported network connection") return } if newHeloName != "" { session.peer.HeloName = newHeloName } if newAddr != nil { tcpAddr.IP = newAddr } if newTCPPort != 0 { tcpAddr.Port = int(newTCPPort) } if newUsername != "" { session.peer.Username = newUsername } if newProto != "" { session.peer.Protocol = newProto } session.welcome() } func parseAddress(src string) (string, error) { if len(src) == 0 || src[0] != '<' || src[len(src)-1] != '>' { return "", fmt.Errorf("Ill-formatted e-mail address: %s", src) } if strings.Count(src, "@") > 1 { return "", fmt.Errorf("Ill-formatted e-mail address: %s", src) } return src[1 : len(src)-1], nil } ================================================ FILE: src/backend/protocols/go.smtp/receiver.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package caliopen_smtp import ( "bytes" broker "github.com/CaliOpen/Caliopen/src/backend/brokers/go.emails" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/Notifications" "time" ) // handler is called by smtpd for each incoming email func (lda *Lda) handler(peer Peer, ev SmtpEnvelope) error { var raw_email bytes.Buffer raw_email.WriteString(string(ev.Data)) emailMessage := EmailMessage{ Email: &Email{ SmtpMailFrom: []string{ev.Sender}, //TODO: handle multiple senders SmtpRcpTo: ev.Recipients, Raw: raw_email, }, Message: &Message{}, } batch := Notifications.NewBatch("smtp") incoming := &broker.SmtpEmail{ EmailMessage: &emailMessage, Response: make(chan *broker.EmailDeliveryAck), Batch: batch, } defer close(incoming.Response) lda.brokerConnectors.Ingress <- incoming select { case response := <-incoming.Response: if response.Err { return Error{ Code: 554, Message: response.Response, } } return nil case <-time.After(30 * time.Second): return Error{ Code: 554, Message: "LDA timeout", } } } ================================================ FILE: src/backend/protocols/go.smtp/server.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package caliopen_smtp import ( log "github.com/Sirupsen/logrus" ) var ( lda *Lda daemon *Server ) // load configuration into package's vars above func InitializeServer(config SMTPConfig) (err error) { lda = new(Lda) daemon = new(Server) err = lda.initialize(config) if err != nil { return } err = daemon.initialize(config) return } // listen & serve func StartServer() { err := lda.start() if err != nil { log.WithError(err).Fatal("LDA failed to start") } else { log.Infof("Caliopen lda started") } err = daemon.start() if err != nil { lda.shutdown() log.WithError(err).Fatal("smtpd failed to start") } log.Infof("Caliopen smtpd started") } func ShutdownServer() (err error) { err = lda.shutdown() if err != nil { log.WithError(err).Warn("Error when shutting down LDA") } err = daemon.shutdown() if err != nil { log.WithError(err).Warn("Error when shutting down smtpd") } return } ================================================ FILE: src/backend/protocols/go.smtp/submitter.go ================================================ // Copyleft (ɔ) 2017 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package caliopen_smtp import ( "bytes" "crypto/tls" "errors" "fmt" broker "github.com/CaliOpen/Caliopen/src/backend/brokers/go.emails" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "gopkg.in/gomail.v2" "io" "net/smtp" "strconv" "strings" "sync" "time" ) type submitter struct { config LDAConfig workersCountMux sync.Mutex runningWorkers int submitChan chan *broker.SmtpEmail } type smtpSender struct { smtpClient d *gomail.Dialer } type smtpClient interface { Hello(string) error Extension(string) (bool, string) StartTLS(*tls.Config) error Auth(smtp.Auth) error Mail(string) error Rcpt(string) error Data() (io.WriteCloser, error) Quit() error Close() error } func (lda *Lda) newSubmitter() (submit *submitter, err error) { submit = &submitter{} submit.config = lda.Config.LDAConfig submit.submitChan = make(chan *broker.SmtpEmail) return } func (lda *Lda) runSubmitterAgent() { for email := range lda.brokerConnectors.Egress { go func(email *broker.SmtpEmail) { lda.outboundListener.workersCountMux.Lock() if lda.outboundListener.runningWorkers < lda.Config.AppConfig.OutWorkers { go lda.OutboundWorker() lda.outboundListener.runningWorkers++ } lda.outboundListener.workersCountMux.Unlock() //submit email lda.outboundListener.submitChan <- email }(email) } } /* OutboundWorker dials to static MTA and maintains connection open to handle outbound deliveries, then close the connection if no email comes in for 30 sec. should be launched in a goroutine */ func (lda *Lda) OutboundWorker() { c := lda.Config.AppConfig d := gomail.NewDialer(c.SubmitAddress, c.SubmitPort, c.SubmitUser, c.SubmitPassword) defer func() { lda.outboundListener.workersCountMux.Lock() lda.outboundListener.runningWorkers-- lda.outboundListener.workersCountMux.Unlock() }() var smtp_sender gomail.SendCloser var smtp_remote_sender gomail.SendCloser var err error open := false for { select { case outcoming, ok := <-lda.outboundListener.submitChan: if !ok { //TODO return } from := outcoming.EmailMessage.Email.SmtpMailFrom[0] //TODO: manage multiple senders to := outcoming.EmailMessage.Email.SmtpRcpTo var raw bytes.Buffer raw.WriteString((&outcoming.EmailMessage.Email.Raw).String()) // send via local or remote MTA, accordingly if outcoming.MTAparams != nil { server := strings.Split(outcoming.MTAparams.Host, ":") host := server[0] port, _ := strconv.Atoi(server[1]) var dialErr error var remoteDialer *gomail.Dialer switch outcoming.MTAparams.AuthType { case Oauth1: dialErr = errors.New("oauth1 mechanism not implemented") case Oauth2: remoteDialer = &gomail.Dialer{ Host: host, Port: port, SSL: false, Auth: &Xoauth2Client{ Username: outcoming.MTAparams.User, Token: outcoming.MTAparams.Password, }, } case LoginPassword: remoteDialer = &gomail.Dialer{ Host: host, Port: port, Username: outcoming.MTAparams.User, Password: outcoming.MTAparams.Password, SSL: port == 465, } default: dialErr = fmt.Errorf("unknown auth mechanism <%s>", outcoming.MTAparams.AuthType) } if dialErr == nil { smtp_remote_sender, dialErr = remoteDialer.Dial() } if dialErr != nil { err = fmt.Errorf("outbound: unable to connect to remote MTA with error : %s", dialErr) } else { err = smtp_remote_sender.Send(from, to, &raw) } } else { // no MTA params means submitter has to go through the configured local MTA if !open { var dialErr error if smtp_sender, dialErr = d.Dial(); dialErr != nil { err = fmt.Errorf("outbound: unable to connect to MTA with error : %s", dialErr) } else { open = true } } if err == nil { err = smtp_sender.Send(from, to, &raw) } } var ack broker.EmailDeliveryAck if err != nil { log.WithError(err).Warn("outbound: unable to send to MTA") ack.Err = true ack.Response = err.Error() } else { ack.Err = false ack.Response = "" } ack.EmailMessage = outcoming.EmailMessage outcoming.Response <- &ack // Close the connection to the SMTP server and this worker // if no email was sent in the last 30 seconds. case <-time.After(30 * time.Second): if open { if err := smtp_sender.Close(); err != nil { log.WithError(err).Warn("outbound: unable to close connection to MTA") } open = false } return } } } ================================================ FILE: src/backend/protocols/go.twitter/account.go ================================================ // Copyleft (ɔ) 2018 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package twitterworker import ( "bytes" "encoding/json" "errors" "fmt" "sort" "strconv" "strings" "time" broker "github.com/CaliOpen/Caliopen/src/backend/brokers/go.twitter" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/Notifications" "github.com/CaliOpen/go-twitter/twitter" log "github.com/Sirupsen/logrus" "github.com/dghubble/oauth1" ) type ( AccountHandler struct { AccountDesk chan uint broker *broker.TwitterBroker closed bool lastDMseen string MasterDesk chan DeskMessage twitterClient *twitter.Client userAccount *TwitterAccount usersScreenNames map[int64]string // a cache facility to avoid calling too often twitter API for screen_name lookup } TwitterAccount struct { accessToken string accessTokenSecret string userID UUID remoteID UUID screenName string twitterID string } ) const ( //AccountDesk commands PollDM = uint(iota) PollTimeLine Stop lastSeenInfosKey = "lastseendm" lastSyncInfosKey = "lastsync" lastErrorKey = "lastFetchError" dateFirstErrorKey = "firstErrorDate" dateLastErrorKey = "lastErrorDate" errorsCountKey = "errorsCount" syncingKey = "syncing" defaultPollInterval = 10 syncingTimeout = 1 // how many hours to wait before restarting sync op ) // NewAccountHandler creates a handler dedicated to a specific twitter account. // It caches remote identity credentials and data, as well as user context connection to twitter API. func NewAccountHandler(userID, remoteID string, worker Worker) (accountHandler *AccountHandler, err error) { accountHandler = new(AccountHandler) accountHandler.AccountDesk = make(chan uint) accountHandler.MasterDesk = worker.Desk b, e := broker.Initialize(worker.Conf.BrokerConfig, worker.Store, worker.Index, worker.NatsConn, worker.Notifier) if e != nil { err = fmt.Errorf("[TwitterAccount]NewAccountHandler failed to initialize a twitter broker : %s", e) return nil, err } accountHandler.broker = b var remote *UserIdentity // retrieve data from db remote, err = accountHandler.broker.Store.RetrieveUserIdentity(userID, remoteID, true) if err != nil { log.WithError(err).Errorf("[TwitterAccount]NewAccountHandler failed to retrieve remote identity <%s> (user <%s>)", remoteID, userID) return } if remote.Credentials == nil { err = fmt.Errorf("[TwitterAccount]NewAccountHandler failed to retrieve credentials for remote identity <%s> (user <%s>)", remoteID, userID) return } accountHandler.userAccount = &TwitterAccount{ accessToken: (*remote.Credentials)["token"], accessTokenSecret: (*remote.Credentials)["secret"], userID: remote.UserId, remoteID: remote.Id, screenName: remote.Identifier, } if lastseen, ok := remote.Infos[lastSeenInfosKey]; ok { accountHandler.lastDMseen = lastseen } else { accountHandler.lastDMseen = "0" } authConf := oauth1.NewConfig(worker.Conf.TwitterAppKey, worker.Conf.TwitterAppSecret) token := oauth1.NewToken(accountHandler.userAccount.accessToken, accountHandler.userAccount.accessTokenSecret) httpClient := authConf.Client(oauth1.NoContext, token) if accountHandler.twitterClient = twitter.NewClient(httpClient); accountHandler.twitterClient == nil { return nil, errors.New("[NewWorker] twitter api failed to create http client") } var twitterID int64 var screenName string accountHandler.usersScreenNames = map[int64]string{} // try to cache account's ID and screenName if twitterid, ok := remote.Infos["twitterid"]; ok && twitterid != "" { accountHandler.userAccount.twitterID = twitterid twitterID, _ = strconv.ParseInt(twitterid, 10, 64) if twittername, ok := remote.Infos["screen_name"]; ok && twittername != "" { screenName = twittername } else { screenName = accountHandler.getAccountName(twitterid) } } else { twitterUser, _, e := accountHandler.twitterClient.Users.Show(&twitter.UserShowParams{ScreenName: accountHandler.userAccount.screenName}) if e == nil { accountHandler.userAccount.twitterID = twitterUser.IDStr twitterID = twitterUser.ID screenName = twitterUser.ScreenName } } if twitterID != 0 && screenName != "" { accountHandler.usersScreenNames[twitterID] = screenName } return } // Start begins infinite loops, until receiving stop order. This func must be call within goroutine. func (worker *AccountHandler) Start() { go func(w *AccountHandler) { for worker.broker != nil { select { case egress, ok := <-w.broker.Connectors.Egress: if !ok { if !worker.closed { close(worker.broker.Connectors.Halt) close(worker.AccountDesk) worker.closed = true } worker.MasterDesk <- DeskMessage{closeAccountOrder, worker} return } err := w.SendDM(egress.Order) if err != nil { egress.Ack <- &DeliveryAck{ Err: true, Response: err.Error(), } } else { egress.Ack <- &DeliveryAck{ Err: false, Response: "OK", } } case <-w.broker.Connectors.Halt: worker.MasterDesk <- DeskMessage{closeAccountOrder, worker} return } } }(worker) for command := range worker.AccountDesk { switch command { case PollDM: worker.PollDM() case Stop: worker.Stop() return default: log.Warnf("worker received unknown command number %d", command) } } } func (worker *AccountHandler) Stop() { if !worker.closed { close(worker.broker.Connectors.Egress) close(worker.broker.Connectors.Halt) close(worker.AccountDesk) worker.closed = true } } // PollDM calls Twitter API endpoint to fetch DMs // it passes unseen DM to its embedded broker func (worker *AccountHandler) PollDM() { // retrieve user_identity.infos accountInfos, retrieveErr := worker.broker.Store.RetrieveRemoteInfosMap(worker.userAccount.userID.String(), worker.userAccount.remoteID.String()) if retrieveErr != nil { log.WithError(retrieveErr).Warnf("[AccountHandler %s] PollDM failed to retrieve infos map", worker.userAccount.remoteID.String()) return } // check if a sync process is running if syncing, ok := accountInfos[syncingKey]; ok && syncing != "" { startDate, e := time.Parse(time.RFC3339, syncing) if e == nil && time.Since(startDate)/time.Hour < syncingTimeout { log.Infof("[PollDM] avoiding concurrent sync for <%s>. Syncing in progress since %s", worker.userAccount.remoteID, accountInfos["syncing"]) return } } // save syncing state in db to prevent concurrent sync accountInfos[syncingKey] = time.Now().Format(time.RFC3339) err := worker.broker.Store.UpdateRemoteInfosMap(worker.userAccount.userID.String(), worker.userAccount.remoteID.String(), accountInfos) if err != nil { log.WithError(err).Infof("[PollDM] failed to update syncing state user <%s>, identity <%s>", worker.userAccount.userID, worker.userAccount.remoteID) return } // do not forget to always write down last_check timestamp // and to remove syncing state before leaving defer func() { if worker.broker != nil { // TODO : remove deletion of errors key from defer func ? (see mastodon broker) delete(accountInfos, lastErrorKey) delete(accountInfos, errorsCountKey) delete(accountInfos, dateFirstErrorKey) delete(accountInfos, dateLastErrorKey) delete(accountInfos, syncingKey) e := worker.broker.Store.UpdateRemoteInfosMap(worker.userAccount.userID.String(), worker.userAccount.remoteID.String(), accountInfos) if e != nil { log.WithError(e).Warnf("[AccountHandler %s] PollDM failed to update sync state in db", worker.userAccount.remoteID.String()) } log.Infof("[AccountHandler %s] PollDM finished", worker.userAccount.remoteID.String()) e = worker.broker.Store.TimestampRemoteLastCheck(worker.userAccount.userID.String(), worker.userAccount.remoteID.String()) if e != nil { log.WithError(e).Warnf("[AccountHandler %s] PollDM failed to update last_check state in db", worker.userAccount.remoteID.String()) } } }() // retrieve DM list from twitter API DMs, _, err := worker.twitterClient.DirectMessages.EventsList(&twitter.DirectMessageEventsListParams{Count: 50}) if err != nil { if e, ok := err.(twitter.APIError); ok { errorsMessages := new(strings.Builder) for _, err := range e.Errors { switch err.Code { case 88: // rate limit error => send throttling order to idpoller var interval int log.Infof("[AccountHandler %s] PollDM : twitter returned rate limit error, slowing down worker for account", worker.userAccount.remoteID) if pollInterval, ok := accountInfos["pollinterval"]; ok { interval, e := strconv.Atoi(pollInterval) if e == nil { interval *= 2 // prevent boundaries overflow : min = 1 min, max = 3 days if interval < 1 || interval > 3*24*60 { interval = defaultPollInterval } } else { interval = defaultPollInterval } } else { interval = defaultPollInterval } newInterval := strconv.Itoa(interval) accountInfos["pollinterval"] = newInterval e := worker.broker.Store.UpdateRemoteInfosMap(worker.userAccount.userID.String(), worker.userAccount.remoteID.String(), accountInfos) if e != nil { log.WithError(e).Warnf("[AccountHandler %s] PollDM : failed to updateRemoteInfosMap with new poll interval", worker.userAccount.userID.String()+"/"+worker.userAccount.remoteID.String()) } order := RemoteIDNatsMessage{ IdentityId: worker.userAccount.remoteID.String(), Order: "update_interval", OrderParam: newInterval, Protocol: "twitter", UserId: worker.userAccount.userID.String(), } jorder, jerr := json.Marshal(order) if jerr == nil { e := worker.broker.NatsConn.Publish(worker.broker.Config.NatsTopicPollerCache, jorder) if e != nil { log.WithError(e).Warnf("[AccountHandler %s] PollDM : failed to publish new poll interval to idpoller", worker.userAccount.userID.String()+"/"+worker.userAccount.remoteID.String()) } } case 89: // invalid token or token expired. Suicide this accountworker thus it will be re-created with new oauth next time idpoller will order it delete(accountInfos, syncingKey) e := worker.saveErrorState(accountInfos, errorsMessages.String()) if e != nil { log.WithError(e).Warnf("[AccountHandler %s] PollDM failed to update sync state in db", worker.userAccount.remoteID.String()) } worker.MasterDesk <- DeskMessage{closeAccountOrder, worker} return } errorsMessages.WriteString(err.Message + " ") } e := worker.saveErrorState(accountInfos, errorsMessages.String()) if e != nil { log.WithError(e).Warnf("[AccountHandler %s] PollDM failed to update sync state in db", worker.userAccount.remoteID.String()) } return } else { e := worker.saveErrorState(accountInfos, err.Error()) if e != nil { log.WithError(e).Warnf("[AccountHandler %s] PollDM failed to update sync state in db", worker.userAccount.remoteID.String()) } return } } sort.Sort(ByAscID(DMs.Events)) // reverse events order to get older DMs first if len(DMs.Events) > 0 && worker.dmNotSeen(DMs.Events[0]) { //TODO: handle pagination with `cursor` param } log.Infof("[AccountHandler %s] PollDM %d events retrieved", worker.userAccount.remoteID.String(), len(DMs.Events)) batch := Notifications.NewBatch("twitter_worker") for _, event := range DMs.Events { if worker.dmNotSeen(event) { //lookup sender & recipient's screen_names because there are not embedded in event object accountName := worker.getAccountName(event.Message.SenderID) if accountName == "" { continue // we don't want to inject a message with an incomplete participant } (*event.Message).SenderScreenName = accountName accountName = worker.getAccountName(event.Message.Target.RecipientID) if accountName == "" { continue // we don't want to inject a message with an incomplete participant } (*event.Message).Target.RecipientScreenName = accountName err = worker.broker.ProcessInDM(worker.userAccount.userID, worker.userAccount.remoteID, &event, true, batch) if err != nil { // something went wrong, forget this DM log.WithError(err).Warnf("[AccountHandler %s] ProcessInDM failed for event : %+v", worker.userAccount.remoteID.String(), event) continue } worker.lastDMseen = event.ID // update sync status in db // TODO: algorithm to shorten pollinterval after new DM has been received accountInfos[lastSeenInfosKey] = event.ID accountInfos[lastSyncInfosKey] = time.Now().Format(time.RFC3339) err = worker.broker.Store.UpdateRemoteInfosMap(worker.userAccount.userID.String(), worker.userAccount.remoteID.String(), accountInfos) if err != nil { log.WithError(err).Warnf("[AccountHandler %s] ProcessInDM failed to update InfosMap for event : %+v", worker.userAccount.remoteID.String(), event) continue } } } batch.Save(worker.broker.Notifier, "", LongLived) } func (worker *AccountHandler) dmNotSeen(event twitter.DirectMessageEvent) bool { return worker.lastDMseen < event.ID } // SendDM delivers DM to Twitter endpoint and give back Twitter's response to broker. func (worker *AccountHandler) SendDM(order BrokerOrder) error { // make use of broker to marshal a direct message brokerPort := make(chan *broker.DMpayload) var brokerMessage *broker.DMpayload go worker.broker.ProcessOutDM(order, brokerPort) select { case brokerMessage = <-brokerPort: if brokerMessage.Err != nil { return brokerMessage.Err } case <-time.After(10 * time.Second): return errors.New("[SendDM] broker timeout") } // retrieve recipient's twitter ID from DM's screenName user, _, userErr := worker.twitterClient.Users.Show(&twitter.UserShowParams{ ScreenName: brokerMessage.DM.Message.Target.RecipientScreenName, }) if userErr != nil { brokerMessage.Response <- broker.TwitterDeliveryAck{ Err: true, Response: userErr.Error(), } return userErr } brokerMessage.DM.Message.Target.RecipientID = user.IDStr // deliver DM through Twitter API createResponse, _, errResponse := worker.twitterClient.DirectMessages.EventsCreate(brokerMessage.DM.Message) if errResponse != nil { brokerMessage.Response <- broker.TwitterDeliveryAck{ Payload: createResponse, Err: true, Response: errResponse.Error(), } return errResponse } // give back Twitter's reply to broker for it finishes its job brokerMessage.Response <- broker.TwitterDeliveryAck{ Payload: createResponse, } select { case brokerMessage = <-brokerPort: if brokerMessage.Err != nil { return brokerMessage.Err } return nil case <-time.After(10 * time.Second): return errors.New("[SendDM] broker timeout") } } // getAccountName returns Twitter account screen name given a Twitter account ID // screen name is retrieve either from worker's cache or Twitter API // returns empty string if it fails. func (worker *AccountHandler) getAccountName(accountID string) (accountName string) { ID, err := strconv.ParseInt(accountID, 10, 64) if err == nil { var inCache bool if accountName, inCache = worker.usersScreenNames[ID]; !inCache { user, resp, err := worker.twitterClient.Users.Show(&twitter.UserShowParams{UserID: ID}) if err == nil && user != nil { (*worker).usersScreenNames[ID] = user.ScreenName return user.ScreenName } else { log.WithError(err).Warnf("[AccountHandler] failed to getAccountName for twitter ID %s. Got user {%+v} and http response {%+v}", accountID, user, resp) } } return accountName } return } // isDMUnique returns true if Twitter Direct Message id is not found within user's messages index // if seeking fails for any reason, true is returned anyway to allow duplication func (worker *AccountHandler) isDMUnique(dmID string) bool { messageID, err := worker.broker.Store.SeekMessageByExternalRef(worker.userAccount.userID.String(), dmID, "") if err != nil || bytes.Equal(messageID.Bytes(), EmptyUUID.Bytes()) { return true } return false } func (worker *AccountHandler) saveErrorState(infos map[string]string, err string) error { // ensure errors data fields are present if _, ok := infos[lastErrorKey]; !ok { infos[lastErrorKey] = "" } if _, ok := infos[dateFirstErrorKey]; !ok { infos[dateFirstErrorKey] = "" } if _, ok := infos[dateLastErrorKey]; !ok { infos[dateLastErrorKey] = "" } if _, ok := infos[errorsCountKey]; !ok { infos[errorsCountKey] = "0" } // log last error infos[lastErrorKey] = "Twitter connection failed : " + err log.Warnf("Twitter connection failed for remote identity %s : %s", worker.userAccount.remoteID, err) // increment counter count, _ := strconv.Atoi(infos[errorsCountKey]) count++ infos[errorsCountKey] = strconv.Itoa(count) // update dates lastDate := time.Now() var firstDate time.Time firstDate, _ = time.Parse(time.RFC3339, infos[dateFirstErrorKey]) if firstDate.IsZero() { firstDate = lastDate } infos[dateFirstErrorKey] = firstDate.Format(time.RFC3339) infos[dateLastErrorKey] = lastDate.Format(time.RFC3339) // check failuresThreshold if lastDate.Sub(firstDate)/time.Hour > failuresThreshold { // disable remote identity err := worker.broker.Store.UpdateUserIdentity(&UserIdentity{ UserId: worker.userAccount.userID, Id: worker.userAccount.remoteID, }, map[string]interface{}{ "Status": "inactive", }) if err != nil { log.WithError(err).Warnf("[saveErrorState] failed to deactivate remote identity %s for user %s", worker.userAccount.remoteID, worker.userAccount.userID) } // send nats message to idpoller to stop polling order := RemoteIDNatsMessage{ IdentityId: worker.userAccount.remoteID.String(), Order: "delete", Protocol: "twitter", UserId: worker.userAccount.userID.String(), } jorder, jerr := json.Marshal(order) if jerr == nil { e := worker.broker.NatsConn.Publish(worker.broker.Config.NatsTopicPollerCache, jorder) if e != nil { log.WithError(e).Warnf("[saveErrorState] failed to publish delete order to idpoller") } } } // udpate UserIdentity in db return worker.broker.Store.UpdateRemoteInfosMap(worker.userAccount.userID.String(), worker.userAccount.remoteID.String(), infos) } // ByAscID implements sort interface type ByAscID []twitter.DirectMessageEvent func (bri ByAscID) Len() int { return len(bri) } func (bri ByAscID) Less(i, j int) bool { return bri[i].ID < bri[j].ID } func (bri ByAscID) Swap(i, j int) { bri[i], bri[j] = bri[j], bri[i] } ================================================ FILE: src/backend/protocols/go.twitter/account_test.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package twitterworker import ( "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/backendstest" "testing" "time" ) func TestNewAccountHandler(t *testing.T) { w, s, err := initWorkerTest() if err != nil { t.Error(err) return } defer s.Shutdown() ah, err := NewAccountHandler(backendstest.EmmaTommeUserId, "b91f0fa8-17a2-4729-8a5a-5ff58ee5c121", *w) if err != nil { t.Error(err) return } if ah.broker == nil { t.Error("expected account handler with broker initialized, got broker==nil") } if ah.twitterClient == nil { t.Error("expected account handler with twitterClient initialized, got client==nil") } if ah.userAccount.twitterID != "000000" { t.Errorf("expected userAccount with twitterID == 000000, got %s", ah.userAccount.twitterID) } // test that closing broker's connectors will kill accountHandler go ah.Start() time.Sleep(100 * time.Millisecond) close(ah.broker.Connectors.Egress) time.Sleep(100 * time.Millisecond) if _, ok := <-ah.AccountDesk; ok { t.Error("expected handler's accountDesk to be closed, still open") } if _, ok := <-ah.broker.Connectors.Halt; ok { t.Error("expected handler.broker's connectors.halt to be closed, still open") } if _, ok := <-ah.broker.Connectors.Egress; ok { t.Error("expected handler.broker's connectors.Egress to be closed, still open") } if len(w.AccountHandlers) > 0 { t.Errorf("expected empty AccountHandlers map, got len=%d", len(w.AccountHandlers)) } } func TestAccountHandler_Start(t *testing.T) { w, s, err := initWorkerTest() if err != nil { t.Error(err) return } defer s.Shutdown() ah, err := NewAccountHandler(backendstest.EmmaTommeUserId, "b91f0fa8-17a2-4729-8a5a-5ff58ee5c121", *w) if err != nil { t.Error(err) return } go ah.Start() } func TestAccountHandler_Stop(t *testing.T) { w, s, err := initWorkerTest() if err != nil { t.Error(err) return } defer s.Shutdown() ah, err := NewAccountHandler(backendstest.EmmaTommeUserId, "b91f0fa8-17a2-4729-8a5a-5ff58ee5c121", *w) if err != nil { t.Error(err) return } go ah.Start() time.Sleep(100 * time.Millisecond) ah.Stop() time.Sleep(100 * time.Millisecond) if _, ok := <-ah.AccountDesk; ok { t.Error("expected handler's accountDesk to be closed, still open") } if _, ok := <-ah.broker.Connectors.Halt; ok { t.Error("expected handler.broker's connectors.halt to be closed, still open") } if _, ok := <-ah.broker.Connectors.Egress; ok { t.Error("expected handler.broker's connectors.Egress to be closed, still open") } } ================================================ FILE: src/backend/protocols/go.twitter/cmd/twitterworker/cli_cmds/root.go ================================================ // Copyleft (ɔ) 2018 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package cmd import ( log "github.com/Sirupsen/logrus" "github.com/spf13/cobra" ) var ( verbose bool version bool RootCmd = &cobra.Command{ Use: "twitterd", Short: "Twitter API daemon", Long: `twitterd is a daemon that subscribes to Twitter accounts on one side and to our NATS queues on other side to executes IO operations with Twitter API`, Run: nil, } ) const __version__ = "0.23.0" func init() { cobra.OnInitialize() RootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "print out more debug information") RootCmd.PersistentFlags().BoolVarP(&version, "version", "V", false, "print out the version of this program") RootCmd.Run = func(cmd *cobra.Command, args []string) { if version { log.Infof("twitterd version %s", __version__) } if len(args) == 0 { cmd.Help() } } RootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { if verbose { log.SetLevel(log.DebugLevel) } else { log.SetLevel(log.InfoLevel) } } RootCmd.AddCommand(versionCmd) } var versionCmd = &cobra.Command{ Use: "version", Short: "Print the version number of twitterd", Long: `All software has versions. This is twitterd's`, Run: func(cmd *cobra.Command, args []string) { log.Infof("twitterd version %s", __version__) }, } ================================================ FILE: src/backend/protocols/go.twitter/cmd/twitterworker/cli_cmds/start.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package cmd import ( "crypto/rand" "fmt" twd "github.com/CaliOpen/Caliopen/src/backend/protocols/go.twitter" log "github.com/Sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" "io" "os" "os/signal" "sync" "syscall" "time" ) const ( shutdownTimeout = 3 // minutes to wait before forcing shutdown ) var ( configPath string configFile string pidFile string signalChannel chan os.Signal twitterWorkers []*twd.Worker startCmd = &cobra.Command{ Use: "start", Short: "Starts a pool of twitter API worker(s)", Run: start, } ) func init() { startCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "twitterworker", "Name of the configuration file, without extension. (YAML, TOML, JSON… allowed)") startCmd.PersistentFlags().StringVarP(&configPath, "configpath", "", "../../../../configs/", "Main config file path.") startCmd.PersistentFlags().StringVarP(&pidFile, "pid-file", "p", "/var/run/caliopen_twitterd.pid", "Path to the pid file") RootCmd.AddCommand(startCmd) signalChannel = make(chan os.Signal, 1) } func start(cmd *cobra.Command, args []string) { var conf twd.WorkerConfig err := readConfig(&conf) if err != nil { log.WithError(err).Fatal("Error while reading config") } // Write out our PID if len(pidFile) > 0 { if f, err := os.Create(pidFile); err == nil { defer f.Close() if _, err := f.WriteString(fmt.Sprintf("%d", os.Getpid())); err == nil { f.Sync() } else { log.WithError(err).Warnf("Error while writing pidFile (%s)", pidFile) } } else { log.WithError(err).Warnf("Error while creating pidFile (%s)", pidFile) } } // init and start worker(s) var i uint8 twitterWorkers = make([]*twd.Worker, conf.Workers) for i = 0; i < conf.Workers; i++ { log.Infof("Initializing Twitter worker %d", i) twitterWorkers[i], err = twd.InitWorker(conf, verbose, randomIdentifier()) if err != nil { log.WithError(err).Fatal("failed to init worker") } go twitterWorkers[i].Start() } // listening mode, waiting for nats orders to add/update workers or os sig to shutdown sigHandler(twitterWorkers) } // ReadConfig which should be called at startup, or when a SIG_HUP is caught func readConfig(config *twd.WorkerConfig) error { // load in the main config. Reading from YAML, TOML, JSON, HCL and Java properties config files v := viper.New() v.SetConfigName(configFile) // name of config file (without extension) v.AddConfigPath(configPath) // path to look for the config file in v.AddConfigPath("$CALIOPENROOT/src/backend/configs/") // call multiple times to add many search paths v.AddConfigPath(".") // optionally look for config in the working directory err := v.ReadInConfig() // Find and read the config file*/ if err != nil { log.WithError(err).Infof("Could not read main config file <%s>.", configFile) return err } err = v.Unmarshal(config) if err != nil { log.WithError(err).Infof("Could not parse config file: <%s>", configFile) return err } return nil } func sigHandler(workers []*twd.Worker) { signal.Notify(signalChannel, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGINT, syscall.SIGKILL) for sig := range signalChannel { if sig == syscall.SIGHUP { // TODO: handle SIGHUP } else if sig == syscall.SIGTERM || sig == syscall.SIGQUIT || sig == syscall.SIGINT || sig == syscall.SIGKILL { log.Infof("Shutdown signal caught. Gracefully halting %d workers within 3 minutes timeframe…", len(workers)) wg := new(sync.WaitGroup) wg.Add(len(workers)) for i := range workers { workers[i].HaltGroup = wg } // timeout mechanism to avoid infinite wait c := make(chan struct{}) go func() { defer close(c) wg.Wait() }() select { case <-c: log.Info("Shutdown completed, exiting") os.Exit(0) case <-time.After(shutdownTimeout * time.Minute): log.Warn("Shutdown timeout, force exiting") os.Exit(0) } } else { os.Exit(0) } } } func randomIdentifier() string { var buf [4]byte _, err := io.ReadFull(rand.Reader, buf[:]) if err != nil { return "00000000" } return fmt.Sprintf("%x", buf[:]) } ================================================ FILE: src/backend/protocols/go.twitter/cmd/twitterworker/main.go ================================================ // Copyleft (ɔ) 2018 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package main import ( "fmt" "github.com/CaliOpen/Caliopen/src/backend/protocols/go.twitter/cmd/twitterworker/cli_cmds" "os" ) func main() { if err := cmd.RootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(-1) } } ================================================ FILE: src/backend/protocols/go.twitter/messaging.go ================================================ // Copyleft (ɔ) 2018 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package twitterworker import ( "encoding/json" "fmt" "github.com/CaliOpen/Caliopen/src/backend/brokers/go.twitter" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" log "github.com/Sirupsen/logrus" "github.com/nats-io/go-nats" "github.com/pkg/errors" "time" ) // WorkerMsgHandler handles message coming from idpoller func (w *Worker) WorkerMsgHandler(msg *nats.Msg) { message := BrokerOrder{} err := json.Unmarshal(msg.Data, &message) if err != nil { log.WithError(err).Errorf("Unable to unmarshal message from NATS. Payload was <%s>", string(msg.Data)) return } switch message.Order { case noPendingJobErr: return case "sync": log.Infof("received sync order for remote twitter ID %s", message.IdentityId) if accountWorker := w.getOrCreateHandler(message.UserId, message.IdentityId); accountWorker != nil { select { case accountWorker.AccountDesk <- PollDM: log.Infof("[DMmsgHandler] ordering to pollDM for remote %s (user %s)", message.IdentityId, message.UserId) case <-time.After(30 * time.Second): log.Warnf("[DMmsgHandler] worker's desk is full for remote %s (user %s)", message.IdentityId, message.UserId) } } else { log.Warnf("[DMmsgHandler] failed to get a worker for remote %s (user %s)", message.IdentityId, message.UserId) w.natsReplyError(msg, errors.New("[DMmsgHandler] failed to get a worker")) } case "reload_worker": log.Infof("received reload_worker order for remote twitter ID %s", message.IdentityId) //TODO: order to force refreshing cache data for an account case "add_worker": log.Infof("received add_worker order for remote twitter ID %s", message.IdentityId) accountWorker := w.getOrCreateHandler(message.UserId, message.IdentityId) if accountWorker == nil { log.WithError(err).Warnf("[WorkerMsgHandler] failed to create new worker for remote %s (user %s)", message.IdentityId, message.UserId) w.natsReplyError(msg, errors.New("[DMmsgHandler] failed to get a worker")) } case "remove_worker": log.Infof("received remove_worker order for remote twitter ID %s", message.IdentityId) // TODO } } // DMmsgHandler handles messages coming on topic dedicated to DM management func (w *Worker) DMmsgHandler(msg *nats.Msg) { message := BrokerOrder{} err := json.Unmarshal(msg.Data, &message) if err != nil { log.WithError(err).Errorf("Unable to unmarshal message from NATS. Payload was <%s>", string(msg.Data)) return } switch message.Order { case "deliver": if accountWorker := w.getOrCreateHandler(message.UserId, message.IdentityId); accountWorker != nil { com := twitter_broker.NatsCom{ Order: message, Ack: make(chan *DeliveryAck), } select { case accountWorker.broker.Connectors.Egress <- com: log.Infof("[DMmsgHandler] sending DM for remote %s (user %s)", message.IdentityId, message.UserId) // non-blocking wait for delivery ack go func(com twitter_broker.NatsCom) { select { case resp := <-com.Ack: if resp.Err { w.natsReplyError(msg, errors.New(resp.Response)) } else { ack := DeliveryAck{ Err: false, Response: "OK", } json_resp, _ := json.Marshal(ack) w.NatsConn.Publish(msg.Reply, json_resp) } case <-time.After(30 * time.Second): w.natsReplyError(msg, errors.New("[DMmsgHandler] timeout waiting broker delivery ack")) } }(com) case <-time.After(30 * time.Second): log.Warnf("[DMmsgHandler] worker's Egress connectors is full for remote %s (user %s)", message.IdentityId, message.UserId) w.natsReplyError(msg, errors.New("[DMmsgHandler] failed to get a worker")) } } else { w.natsReplyError(msg, errors.New("[DMmsgHandler] failed to get a worker")) } default: w.natsReplyError(msg, errors.New("not implemented")) } } func (w *Worker) natsReplyError(msg *nats.Msg, err error) { log.WithError(err).Warnf("twitter broker [outbound] : error when processing incoming nats message : %v", *msg) ack := DeliveryAck{ Err: true, Response: fmt.Sprintf("failed to send message with error « %s » ", err), //TODO } json_resp, _ := json.Marshal(ack) w.NatsConn.Publish(msg.Reply, json_resp) } ================================================ FILE: src/backend/protocols/go.twitter/messaging_test.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package twitterworker import ( "encoding/json" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/backendstest" idpoller "github.com/CaliOpen/Caliopen/src/backend/workers/go.remoteIDs" "github.com/nats-io/go-nats" "github.com/satori/go.uuid" "testing" "time" ) func TestWorker_WorkerMsgHandler(t *testing.T) { w, s, err := initWorkerTest() if err != nil { t.Error(err) return } defer s.Shutdown() noJobMsg := nats.Msg{ Subject: "test", Reply: "testMsgReply", Data: []byte(`{"order":"no pending job"}`), } // 'no job' message should trigger an immediate return c := make(chan struct{}) go func() { w.WorkerMsgHandler(&noJobMsg) close(c) }() select { case <-c: case <-time.After(100 * time.Millisecond): t.Error("timeout waiting for no job return") } gotReply := false w.NatsConn.Subscribe("testMsgReply", func(msg *nats.Msg) { gotReply = true }) // test 'add_worker' that should fail fakeID := uuid.NewV4().String() job := idpoller.Job{ Worker: "twitter", Order: BrokerOrder{ MessageId: fakeID, Order: "add_worker", IdentityId: fakeID, UserId: fakeID, }, } data, _ := json.Marshal(job.Order) syncMsg := nats.Msg{ Subject: "test", Reply: "testMsgReply", Data: data, } w.WorkerMsgHandler(&syncMsg) time.Sleep(time.Second) if !gotReply { t.Error("expected worker replied an error for bad 'add_worker', got nothing on topic") } // test 'sync' order that should fail gotReply = false job = idpoller.Job{ Worker: "twitter", Order: BrokerOrder{ MessageId: fakeID, Order: "sync", IdentityId: fakeID, UserId: fakeID, }, } data, _ = json.Marshal(job.Order) syncMsg = nats.Msg{ Subject: "test", Reply: "testMsgReply", Data: data, } w.WorkerMsgHandler(&syncMsg) time.Sleep(time.Second) if !gotReply { t.Error("expected worker replied an error for bad 'sync', got nothing on topic") } // test 'add_worker' with a valid remote gotReply = false job = idpoller.Job{ Worker: "twitter", Order: BrokerOrder{ MessageId: fakeID, Order: "add_worker", IdentityId: "b91f0fa8-17a2-4729-8a5a-5ff58ee5c121", UserId: backendstest.EmmaTommeUserId, }, } data, _ = json.Marshal(job.Order) syncMsg = nats.Msg{ Subject: "test", Reply: "testMsgReply", Data: data, } w.NatsConn.Subscribe("testMsgReply", func(msg *nats.Msg) { t.Errorf("expected no reply for valid 'add_worker' order , got %s", msg.Data) }) w.WorkerMsgHandler(&syncMsg) // test 'sync' order with a valid remote job = idpoller.Job{ Worker: "twitter", Order: BrokerOrder{ MessageId: fakeID, Order: "sync", IdentityId: "b91f0fa8-17a2-4729-8a5a-5ff58ee5c121", UserId: backendstest.EmmaTommeUserId, }, } data, _ = json.Marshal(job.Order) syncMsg = nats.Msg{ Subject: "test", Reply: "testMsgReply", Data: data, } w.NatsConn.Subscribe("testMsgReply", func(msg *nats.Msg) { t.Errorf("expected no reply for valid 'sync' order , got %s", msg.Data) }) w.WorkerMsgHandler(&syncMsg) } func TestWorker_DMmsgHandler(t *testing.T) { w, s, err := initWorkerTest() if err != nil { t.Error(err) return } defer s.Shutdown() gotReply := false w.NatsConn.Subscribe("testMsgReply", func(msg *nats.Msg) { gotReply = true }) // test 'deliver' that should fail fakeID := uuid.NewV4().String() job := idpoller.Job{ Worker: "twitter", Order: BrokerOrder{ MessageId: fakeID, Order: "deliver", IdentityId: fakeID, UserId: fakeID, }, } data, _ := json.Marshal(job.Order) deliverMsg := nats.Msg{ Subject: "test", Reply: "testMsgReply", Data: data, } w.DMmsgHandler(&deliverMsg) time.Sleep(time.Second) if !gotReply { t.Error("expected worker replied an error for bad 'deliver', got nothing on topic") } // test 'deliver' with a valid remote job = idpoller.Job{ Worker: "twitter", Order: BrokerOrder{ MessageId: fakeID, Order: "deliver", IdentityId: "b91f0fa8-17a2-4729-8a5a-5ff58ee5c121", UserId: backendstest.EmmaTommeUserId, }, } data, _ = json.Marshal(job.Order) deliverMsg = nats.Msg{ Subject: "test", Reply: "testMsgReply", Data: data, } w.NatsConn.Subscribe("testMsgReply", func(msg *nats.Msg) { // should return an error because deliver process is not mocked for now gotReply = true var reply DeliveryAck err = json.Unmarshal(msg.Data, &reply) if !reply.Err { t.Error("expected 'deliver' order to trigger an error, got reply.Err == false") } }) w.DMmsgHandler(&deliverMsg) time.Sleep(time.Second) if !gotReply { t.Error("expected worker replied an error for valid 'deliver', got nothing on topic") } } ================================================ FILE: src/backend/protocols/go.twitter/worker.go ================================================ package twitterworker import ( "errors" "fmt" broker "github.com/CaliOpen/Caliopen/src/backend/brokers/go.twitter" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/index/elasticsearch" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/cassandra" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/Notifications" log "github.com/Sirupsen/logrus" "github.com/gocql/gocql" "github.com/nats-io/go-nats" "sync" "time" ) type ( Worker struct { AccountHandlers map[string]*AccountHandler // one worker per active Twitter account HaltGroup *sync.WaitGroup Index backends.LDAIndex Id string NatsConn *nats.Conn NatsSubs []*nats.Subscription Notifier *Notifications.Notifier Store backends.LDAStore WorkersGuard *sync.RWMutex Conf WorkerConfig Desk chan DeskMessage // chan to allow accountworkers to communicate with their master } WorkerConfig struct { Workers uint8 `mapstructure:"workers"` TwitterAppKey string `mapstructure:"twitter_app_key"` TwitterAppSecret string `mapstructure:"twitter_app_secret"` BrokerConfig broker.BrokerConfig `mapstructure:"BrokerConfig"` } DeskMessage struct { order string account *AccountHandler } ) const ( failuresThreshold = 72 // how many hours to wait before disabling a faulty remote. noPendingJobErr = "no pending job" pollThrottling = 10 * time.Second needJobOrderStr = `{"worker":"%s","order":{"order":"need_job"}}` closeAccountOrder = "close_account" ) func InitWorker(conf WorkerConfig, verboseLog bool, id string) (worker *Worker, err error) { if verboseLog { log.SetLevel(log.DebugLevel) } worker = &Worker{ AccountHandlers: map[string]*AccountHandler{}, Conf: conf, Id: id, WorkersGuard: new(sync.RWMutex), } // init Store switch conf.BrokerConfig.StoreName { case "cassandra": c := store.CassandraConfig{ Hosts: conf.BrokerConfig.StoreConfig.Hosts, Keyspace: conf.BrokerConfig.StoreConfig.Keyspace, Consistency: gocql.Consistency(conf.BrokerConfig.StoreConfig.Consistency), SizeLimit: conf.BrokerConfig.StoreConfig.SizeLimit, UseVault: conf.BrokerConfig.StoreConfig.UseVault, } if conf.BrokerConfig.StoreConfig.ObjectStore == "s3" { c.WithObjStore = true c.Endpoint = conf.BrokerConfig.StoreConfig.OSSConfig.Endpoint c.AccessKey = conf.BrokerConfig.StoreConfig.OSSConfig.AccessKey c.SecretKey = conf.BrokerConfig.StoreConfig.OSSConfig.SecretKey c.RawMsgBucket = conf.BrokerConfig.StoreConfig.OSSConfig.Buckets["raw_messages"] c.AttachmentBucket = conf.BrokerConfig.StoreConfig.OSSConfig.Buckets["temporary_attachments"] c.Location = conf.BrokerConfig.StoreConfig.OSSConfig.Location } if conf.BrokerConfig.StoreConfig.UseVault { c.HVaultConfig.Url = conf.BrokerConfig.StoreConfig.VaultConfig.Url c.HVaultConfig.Username = conf.BrokerConfig.StoreConfig.VaultConfig.Username c.HVaultConfig.Password = conf.BrokerConfig.StoreConfig.VaultConfig.Password } b, e := store.InitializeCassandraBackend(c) if e != nil { err = e log.WithError(err).Warnf("[TwitterWorker] initialization of %s backend failed", conf.BrokerConfig.StoreName) return } worker.Store = backends.LDAStore(b) // type conversion to LDA interface default: log.Warnf("[TwitterWorker] unknown store backend: %s", conf.BrokerConfig.StoreName) err = errors.New("[TwitterWorker] unknown store backend") return } // init Index switch conf.BrokerConfig.LDAConfig.IndexName { case "elasticsearch": c := index.ElasticSearchConfig{ Urls: conf.BrokerConfig.LDAConfig.IndexConfig.Urls, } i, e := index.InitializeElasticSearchIndex(c) if e != nil { err = e log.WithError(err).Warnf("[TwitterBroker] initialization of %s backend failed", conf.BrokerConfig.IndexName) return } worker.Index = backends.LDAIndex(i) // type conversion to LDA interface default: log.Warnf("[TwitterBroker] unknown index backend: %s", conf.BrokerConfig.LDAConfig.IndexName) err = errors.New("[TwitterBroker] unknown index backend") return } worker.NatsConn, err = nats.Connect(conf.BrokerConfig.NatsURL) if err != nil { log.WithError(err).Warn("[TwitterBroker] initalization of NATS connexion failed") return } caliopenConfig := CaliopenConfig{ NotifierConfig: conf.BrokerConfig.LDAConfig.NotifierConfig, NatsConfig: NatsConfig{ Url: conf.BrokerConfig.NatsURL, }, RESTstoreConfig: RESTstoreConfig{ BackendName: conf.BrokerConfig.StoreName, Consistency: conf.BrokerConfig.StoreConfig.Consistency, Hosts: conf.BrokerConfig.StoreConfig.Hosts, Keyspace: conf.BrokerConfig.StoreConfig.Keyspace, OSSConfig: conf.BrokerConfig.StoreConfig.OSSConfig, ObjStoreType: conf.BrokerConfig.StoreConfig.ObjectStore, SizeLimit: conf.BrokerConfig.StoreConfig.SizeLimit, }, RESTindexConfig: RESTIndexConfig{ Hosts: conf.BrokerConfig.LDAConfig.IndexConfig.Urls, IndexName: conf.BrokerConfig.LDAConfig.IndexName, }, } worker.Notifier = Notifications.NewNotificationsFacility(caliopenConfig, worker.NatsConn) // init Nats connector worker.NatsConn, err = nats.Connect(conf.BrokerConfig.NatsURL) if err != nil { log.WithError(err).Fatal("[TwitterWorker] initialization of NATS connexion failed") } worker.NatsSubs = make([]*nats.Subscription, 1) worker.NatsSubs[0], err = worker.NatsConn.QueueSubscribe(conf.BrokerConfig.NatsTopicDMs, conf.BrokerConfig.NatsQueue, worker.DMmsgHandler) if err != nil { log.WithError(err).Fatal("[TwitterWorker] initialization of NATS fetcher subscription failed") } err = worker.NatsConn.Flush() if err != nil { log.WithError(err).Fatal("[TwitterWorker] initialization of NATS fetcher subscription failed") } worker.Desk = make(chan DeskMessage, 2) return worker, nil } func (worker *Worker) Start(throttling ...time.Duration) { var throttle time.Duration if len(throttling) == 1 && throttling[0] != 0 { throttle = throttling[0] } else { throttle = pollThrottling } go func() { for msg := range worker.Desk { switch msg.order { case closeAccountOrder: if msg.account != nil { worker.RemoveAccountHandler(msg.account) } default: log.Debugf("[TwitterWorker] received unknown order « %s » from account %s", msg.order, msg.account.userAccount.userID.String()+msg.account.userAccount.remoteID.String()) } } }() // start throttled jobs polling log.Infof("Twitter worker %s starting with %d sec throttling", worker.Id, throttle/time.Second) for { start := time.Now() requestOrder := []byte(fmt.Sprintf(needJobOrderStr, worker.Id)) log.Infof("Twitter worker %s is requesting jobs to idpoller", worker.Id) resp, err := worker.NatsConn.Request(worker.Conf.BrokerConfig.NatsTopicPoller, requestOrder, time.Minute) if err != nil { log.WithError(err).Warnf("[worker %s] failed to request pending jobs on nats", worker.Id) } else { worker.WorkerMsgHandler(resp) } // check for interrupt after job is finished if worker.HaltGroup != nil { worker.stop() break } elapsed := time.Now().Sub(start) if elapsed < throttle { time.Sleep(throttle - elapsed) } } } func (worker *Worker) stop() { for _, w := range worker.AccountHandlers { w.AccountDesk <- Stop } for _, sub := range worker.NatsSubs { sub.Unsubscribe() } worker.NatsConn.Close() worker.Store.Close() worker.Index.Close() worker.HaltGroup.Done() close(worker.Desk) log.Infof("worker %s stopped", worker.Id) } // getOrCreateHandler returns a pointer to a worker already in cache // or tries to create a new worker for the remote identity if not. // returns nil if get or create failed. func (w *Worker) getOrCreateHandler(userId, remoteId string) *AccountHandler { w.WorkersGuard.RLock() if accountHandler, ok := w.AccountHandlers[userId+remoteId]; ok { w.WorkersGuard.RUnlock() return accountHandler } else { w.WorkersGuard.RUnlock() log.Infof("[getOrCreateHandler] failed to retrieve registered worker for remote %s (user %s). Trying to add one.", remoteId, userId) if userId == "" || remoteId == "" { return nil } accountHandler, err := NewAccountHandler(userId, remoteId, *w) if err != nil { log.WithError(err).Warnf("[getOrCreateHandler] failed to create new worker for remote %s (user %s)", remoteId, userId) return nil } w.RegisterAccountHandler(accountHandler) go accountHandler.Start() return accountHandler } } func (w *Worker) RegisterAccountHandler(accountHandler *AccountHandler) { workerKey := accountHandler.userAccount.userID.String() + accountHandler.userAccount.remoteID.String() // stop & remove handler first if it's already registered w.WorkersGuard.RLock() registeredHandler, ok := w.AccountHandlers[workerKey] w.WorkersGuard.RUnlock() if ok { w.RemoveAccountHandler(registeredHandler) } w.WorkersGuard.Lock() w.AccountHandlers[workerKey] = accountHandler w.WorkersGuard.Unlock() } func (w *Worker) RemoveAccountHandler(accountHandler *AccountHandler) { workerKey := accountHandler.userAccount.userID.String() + accountHandler.userAccount.remoteID.String() w.WorkersGuard.Lock() accountHandler.Stop() delete(w.AccountHandlers, workerKey) w.WorkersGuard.Unlock() } ================================================ FILE: src/backend/protocols/go.twitter/worker_test.go ================================================ // Copyleft (ɔ) 2019 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package twitterworker import ( "encoding/json" "fmt" "github.com/CaliOpen/Caliopen/src/backend/brokers/go.twitter" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/backendstest" "github.com/nats-io/gnatsd/server" "github.com/nats-io/go-nats" "github.com/phayes/freeport" "github.com/satori/go.uuid" "math/rand" "strconv" "sync" "testing" "time" ) const ( natsUrl = "0.0.0.0" ) func initWorkerTest() (worker *Worker, natsServer *server.Server, err error) { // starting an embedded nats server port, err := freeport.GetFreePort() if err != nil { return nil, nil, err } natsServer, err = server.NewServer(&server.Options{ Host: natsUrl, Port: port, HTTPPort: -1, Cluster: server.ClusterOpts{Port: -1}, NoLog: true, NoSigs: true, Debug: false, Trace: false, }) if err != nil || natsServer == nil { panic(fmt.Sprintf("No NATS Server object returned: %v", err)) } go natsServer.Start() // Wait for accept loop(s) to be started if !natsServer.ReadyForConnections(10 * time.Second) { panic("Unable to start NATS Server in Go Routine") } worker = &Worker{ AccountHandlers: map[string]*AccountHandler{}, Conf: WorkerConfig{ BrokerConfig: twitter_broker.BrokerConfig{ NatsQueue: "Twitterworkers", NatsTopicPoller: "twitterJobs", NatsTopicPollerCache: "idCache", NatsTopicDMs: "twitter_dm", }, }, Id: "worker_id", Index: backendstest.GetLDAIndexBackend(), Store: backendstest.GetLDAStoreBackend(), WorkersGuard: new(sync.RWMutex), } worker.NatsConn, err = nats.Connect("nats://" + natsUrl + ":" + strconv.Itoa(port)) if err != nil { return nil, nil, fmt.Errorf("[initMqHandler] failed to init NATS connection : %s", err) } worker.NatsSubs = make([]*nats.Subscription, 1) worker.NatsSubs[0], err = worker.NatsConn.QueueSubscribe(worker.Conf.BrokerConfig.NatsTopicDMs, worker.Conf.BrokerConfig.NatsQueue, worker.DMmsgHandler) if err != nil { return nil, nil, err } worker.Desk = make(chan DeskMessage) go func() { for msg := range worker.Desk { switch msg.order { case closeAccountOrder: if msg.account != nil { worker.RemoveAccountHandler(msg.account) } } } }() return } func TestWorker_StartAndStop(t *testing.T) { w, s, err := initWorkerTest() if err != nil { t.Error(err) return } defer s.Shutdown() // test if worker requests on Nats every second with the right payload c := make(chan struct{}) wg := new(sync.WaitGroup) wg.Add(1) go w.Start(time.Second) count := 0 _, err = w.NatsConn.Subscribe("twitterJobs", func(msg *nats.Msg) { var req WorkerRequest err := json.Unmarshal(msg.Data, &req) if err != nil { t.Errorf("unable to unmarshal worker's request : %s", err) return } if req.Order.Order != "need_job" { t.Errorf("expected to receive order 'need_job', got %s", req.Order.Order) } count++ w.NatsConn.Publish(msg.Reply, []byte(`{"order":"no pending job"}`)) if count == 3 { wg.Done() return } }) if err != nil { t.Error(err) } go func() { wg.Wait() close(c) }() select { case <-c: // worker confirmed to send request every second ; now test halting w.HaltGroup = new(sync.WaitGroup) w.HaltGroup.Add(1) time.Sleep(500 * time.Millisecond) if !w.NatsConn.IsClosed() { t.Error("expected worker's nats connexion to be closed") } for _, sub := range w.NatsSubs { if sub.IsValid() { t.Errorf("expected all worker's subscription closed, got <%s> still valid", sub.Subject) } } return case <-time.After(5 * time.Second): t.Error("timeout waiting for twitter worker to send requests on nats") } } func TestWorker_RegisterAccountHandler(t *testing.T) { w, s, err := initWorkerTest() if err != nil { t.Error(err) return } defer s.Shutdown() // test concurrent account handler registration const count = 1000 // must be an even number workers := [count + 1]string{} wg := new(sync.WaitGroup) wg.Add(count) c := make(chan struct{}) for i := 0; i < count; i++ { go func(indice int) { handler := new(AccountHandler) userId := UUID(uuid.NewV4()) remoteId := UUID(uuid.NewV4()) workers[indice] = userId.String() + remoteId.String() handler.userAccount = &TwitterAccount{ remoteID: remoteId, twitterID: strconv.Itoa(indice), userID: userId, } handler.broker = &twitter_broker.TwitterBroker{ NatsConn: w.NatsConn, Store: backendstest.GetLDAStoreBackend(), Index: backendstest.GetLDAIndexBackend(), Connectors: twitter_broker.TwitterBrokerConnectors{ Egress: make(chan twitter_broker.NatsCom), Halt: make(chan struct{}), }, } handler.AccountDesk = make(chan uint) w.RegisterAccountHandler(handler) wg.Done() }(i) } go func() { wg.Wait() close(c) }() select { case <-c: if len(w.AccountHandlers) != count { t.Errorf("expected %d accountHandlers, got %d", count, len(w.AccountHandlers)) } // pick few handlers randomly to test AccountHandlers consistency rand.Seed(time.Now().Unix()) for i := 0; i < count/2; i++ { pick := rand.Intn(count) if handler, ok := w.AccountHandlers[workers[pick]]; ok { handlerKey := handler.userAccount.userID.String() + handler.userAccount.remoteID.String() if handlerKey != workers[pick] { t.Errorf("expected handler's key to be %s, got %s", workers[pick], handlerKey) } } else { t.Errorf("expected to find handler with key %s, got nothing", workers[pick]) } } // re-register an existing handler to test stop&remove operations pick := rand.Intn(count) handler := w.AccountHandlers[workers[pick]] const s = "register twice" (*handler).userAccount.screenName = s w.RegisterAccountHandler(handler) if w.AccountHandlers[workers[pick]].userAccount.screenName != s { t.Errorf("expected userAccount.screenName of re-registered worker to be %s, got %s", s, w.AccountHandlers[workers[pick]].userAccount.screenName) } case <-time.After(time.Second): t.Error("timeout waiting for concurrent RegisterAccountHandler") } } func TestWorker_getOrCreateHandler(t *testing.T) { w, s, err := initWorkerTest() if err != nil { t.Error(err) return } defer s.Shutdown() // test automatic creation of AccountHandler userId := backendstest.EmmaTommeUserId remoteId := "b91f0fa8-17a2-4729-8a5a-5ff58ee5c121" handler := w.getOrCreateHandler(userId, remoteId) if handler == nil { t.Error("expected a new handler, got nil") } } func TestWorker_RemoveAccountHandler(t *testing.T) { w, s, err := initWorkerTest() if err != nil { t.Error(err) return } defer s.Shutdown() // add a bunch of workers const count = 1000 // must be an even number workers := [count + 1][2]string{} for i := 0; i < count; i++ { handler := new(AccountHandler) userId := UUID(uuid.NewV4()) remoteId := UUID(uuid.NewV4()) workers[i] = [2]string{userId.String(), remoteId.String()} handler.userAccount = &TwitterAccount{ remoteID: remoteId, twitterID: strconv.Itoa(i), userID: userId, } handler.broker = &twitter_broker.TwitterBroker{ NatsConn: w.NatsConn, Store: backendstest.GetLDAStoreBackend(), Index: backendstest.GetLDAIndexBackend(), Connectors: twitter_broker.TwitterBrokerConnectors{ Egress: make(chan twitter_broker.NatsCom), Halt: make(chan struct{}), }, } handler.AccountDesk = make(chan uint) w.RegisterAccountHandler(handler) } // concurrently get & remove half of workers wg := new(sync.WaitGroup) wg.Add(count / 2) c := make(chan struct{}) for i := 0; i < count/2; i++ { go func(indice int) { ah := w.getOrCreateHandler(workers[indice][0], workers[indice][1]) if ah != nil { w.RemoveAccountHandler(ah) } else { t.Error("expcted to get a worker, got nil") } wg.Done() }(i) } go func() { wg.Wait() close(c) }() select { case <-c: if len(w.AccountHandlers) != count/2 { t.Errorf("expected %d account handlers left, got %d", count/2, len(w.AccountHandlers)) } case <-time.After(time.Second): t.Error("timeout waiting for concurrent RemoveAccountHandler") } } ================================================ FILE: src/backend/tools/go.CLI/cmd/gocaliopen/cli_cmds/changeIdentitiyEmailProtocol.go ================================================ // Copyleft (ɔ) 2018 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package cmd import ( "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/cassandra" log "github.com/Sirupsen/logrus" "github.com/gocql/gocql" "github.com/spf13/cobra" ) var emailProtocolCmd = &cobra.Command{ Use: "changeIdentityEmailProtocol", Short: "rename protocol from `imap`|`smtp` to `email` in UserIdentity", Long: `command updates protocol name for email to conform to new specs`, Run: emailProtocolMigration, } func init() { RootCmd.AddCommand(emailProtocolCmd) } func emailProtocolMigration(cmd *cobra.Command, args []string) { var err error //check/get connexions to facilities //store var Store *store.CassandraBackend Store, err = getStoreFacility() if err != nil { log.WithError(err).Fatalf("initialization of %s backend failed", apiConf.BackendName) } defer Store.Close() /* get all identities */ erroneousUsers := []string{} if lookups, err := Store.Session.Query(`SELECT user_id, identity_id, protocol FROM user_identity`).Iter().SliceMap(); err != nil { log.WithError(err).Fatal("failed to retrieve remote identities") } else { log.Infof("\nFound %d identities to work with\n", len(lookups)) for _, lookup := range lookups { if lookup["protocol"].(string) == objects.SmtpProtocol || lookup["protocol"].(string) == objects.ImapProtocol { var userId string var remoteId string userId = lookup["user_id"].(gocql.UUID).String() remoteId = lookup["identity_id"].(gocql.UUID).String() userIdentity, err := Store.RetrieveUserIdentity(userId, remoteId, true) if err != nil { log.WithError(err).Warnf("failed to retrieve identity <%s> for user <%s>", userId, remoteId) continue } updateFields := map[string]interface{}{} userIdentity.Protocol = objects.EmailProtocol updateFields["Protocol"] = objects.EmailProtocol err = Store.UpdateUserIdentity(userIdentity, updateFields) if err != nil { log.Warnf("failed to update identity <%s> for user <%s>", userIdentity.Id.String(), userIdentity.UserId.String()) } else { Store.Session.Query(`DELETE FROM identity_lookup WHERE identifier = ? AND protocol = ? AND user_id = ?`, userIdentity.Identifier, lookup["protocol"].(string), userIdentity) } } } log.Info("\nAll done\n") if len(erroneousUsers) > 0 { log.Warnf("Users with errors : %v", erroneousUsers) } } } ================================================ FILE: src/backend/tools/go.CLI/cmd/gocaliopen/cli_cmds/changeUserIdentitiesCredentialsKeys.go ================================================ // Copyleft (ɔ) 2018 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package cmd import ( "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/cassandra" log "github.com/Sirupsen/logrus" "github.com/gocql/gocql" "github.com/spf13/cobra" ) var credentialsKeysMigrationCmd = &cobra.Command{ Use: "credentialsKeysMigration", Short: "rename keys in UserIdentity.Infos and .Credentials to conform to new specs", Long: `command updates keys name to conform to new specs after branch "imap-outbound" has been merged in prod"`, Run: credentialsKeysMigration, } func init() { RootCmd.AddCommand(credentialsKeysMigrationCmd) } func credentialsKeysMigration(cmd *cobra.Command, args []string) { var err error //check/get connexions to facilities //store var Store *store.CassandraBackend Store, err = getStoreFacility() if err != nil { log.WithError(err).Fatalf("initialization of %s backend failed", apiConf.BackendName) } defer Store.Close() /* get all remote_identities */ erroneousUsers := []string{} if lookups, err := Store.Session.Query(`SELECT * FROM identity_type_lookup WHERE type='remote'`).Iter().SliceMap(); err != nil { log.WithError(err).Fatal("failed to retrieve remote identities") } else { log.Infof("\nFound %d identities to work with\n", len(lookups)) for _, lookup := range lookups { var userId string var remoteId string userId = lookup["user_id"].(gocql.UUID).String() remoteId = lookup["identity_id"].(gocql.UUID).String() userIdentity, err := Store.RetrieveUserIdentity(userId, remoteId, true) if err != nil { log.WithError(err).Warnf("failed to retrieve identity <%s> for user <%s>", userId, remoteId) continue } updateFields := map[string]interface{}{} updateInfos := map[string]string{} updateCredentials := objects.Credentials{} //copy values from infos' map but `server` key for k,v := range userIdentity.Infos { if k == "server" { updateInfos["inserver"] = v } else if k != "inserver"{ updateInfos[k] = v } } //copy values from credentials' map but `password` and `username` if userIdentity.Credentials != nil { for k, v := range *userIdentity.Credentials { if k == "password" { updateCredentials["inpassword"] = v } else if k == "username" { updateCredentials["inusername"] = v } else { updateCredentials[k] = v } } } updateFields["Infos"] = updateInfos updateFields["Credentials"] = &updateCredentials err = Store.UpdateUserIdentity(userIdentity, updateFields) if err != nil { log.Warnf("failed to update identity <%s> for user <%s>", userIdentity.Id.String(), userIdentity.UserId.String()) } } log.Info("\nAll done\n") if len(erroneousUsers) > 0 { log.Warnf("Users with errors : %v", erroneousUsers) } } } ================================================ FILE: src/backend/tools/go.CLI/cmd/gocaliopen/cli_cmds/fixMissingParticipants.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package cmd import ( "context" "encoding/json" "errors" "fmt" "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/index/elasticsearch" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/cassandra" log "github.com/Sirupsen/logrus" "github.com/gocql/gocql" "github.com/nats-io/go-nats" "github.com/spf13/cobra" "os" "strings" "time" ) // fixMissingParticipantsCmd represents the fixMissingParticipants command var fixMissingParticipantsCmd = &cobra.Command{ Use: "fixMissingParticipants", Short: "Fill missing message.participants in db and index", Long: `This command iterates over all messages in cassandra to find messages that miss participants. If message.participants exists in index, db version is filled with it. If not, command will reinject raw_message in stack to trigger again message inbound processing.`, Run: fixMissingParticipants, } func init() { RootCmd.AddCommand(fixMissingParticipantsCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // fixMissingParticipantsCmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: // fixMissingParticipantsCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } func fixMissingParticipants(cmd *cobra.Command, args []string) { var err error //check/get connexions to facilities //store var Store *store.CassandraBackend Store, err = getStoreFacility() if err != nil { log.WithError(err).Fatalf("initialization of %s backend failed", apiConf.BackendName) } defer Store.Close() //index var Index *index.ElasticSearchBackend Index, err = getIndexFacility() if err != nil { log.WithError(err).Fatalf("initialization of %s index failed", apiConf.IndexConfig.IndexName) } defer Index.Close() //nats var MsgQueue *nats.Conn MsgQueue, err = getMsgSystemFacility() if err != nil { log.WithError(err).Fatal("initiazation of message queue failed") } var total int count := 0 invalidCount := 0 fixedCount := 0 failedCount := 0 // handleInvalidMessage := func(msgId, usrId, rawMsgId gocql.UUID) { invalidCount++ //try to fetch message from index and check if it has participants result, err := Index.Client.Get().Index(usrId.String()).Type(objects.MessageIndexType).Id(msgId.String()).Do(context.TODO()) if err != nil && !strings.Contains(err.Error(), "Not Found") { log.Warn(err) failedCount++ return } if result != nil && result.Found { msg := new(objects.Message).NewEmpty().(*objects.Message) if err := json.Unmarshal(*result.Source, msg); err != nil { log.Warn(err) failedCount++ return } err = msg.Message_id.UnmarshalBinary(msgId.Bytes()) if err != nil { log.WithError(err).Warn("failed to unmarshal messageId") failedCount++ return } if msg.Participants != nil && len(msg.Participants) > 0 { err = Store.UpdateMessage(msg, map[string]interface{}{ "Participants": msg.Participants, }) if err != nil { log.WithError(err).Warnf("failed to update message in db") failedCount++ return } fixedCount++ return } } log.Infoln("failed to retrieve participant from index, trying to reinject raw message") var newMsgId string newMsgId, err = reInjectRaw(usrId.String(), rawMsgId.String(), MsgQueue) if err != nil { log.WithError(err).Warn("failed to re-inject raw message in stack") failedCount++ return } //reinjection OK, get previous status and delete former invalid message formerMsg, err := Store.RetrieveMessage(usrId.String(), msgId.String()) if err == nil { newMsg, err := Store.RetrieveMessage(usrId.String(), newMsgId) if err == nil { err = Store.UpdateMessage(newMsg, map[string]interface{}{ "Date_sort": formerMsg.Date_sort, "Is_answered": formerMsg.Is_answered, "Is_unread": formerMsg.Is_unread, "Tags": formerMsg.Tags, }) if err != nil { log.WithError(err).Warnf("failed to update new message with former status %s", msgId) } } } err = Store.Session.Query(`DELETE FROM message WHERE message_id = ? AND user_id = ?`, msgId, usrId).Exec() if err != nil { log.WithError(err).Warnf("failed to delete former invalid message %s", msgId) } fixedCount++ } //get an iterator on table message and iterate over all messages Store.Session.SetTrace(gocql.NewTraceWriter(Store.Session, os.Stdout)) Store.Session.Query(`SELECT count(*) FROM message`).Scan(&total) msgIterator := Store.Session.Query(`SELECT message_id, user_id, raw_msg_id, participants FROM message`).PageSize(500).NoSkipMetadata().Iter() if msgIterator == nil { log.Fatal("fail to get iterator on table message") } var msgId gocql.UUID var usrId gocql.UUID var rawMsgId gocql.UUID var participants []map[string]interface{} for msgIterator.Scan(&msgId, &usrId, &rawMsgId, &participants) { count++ if len(msgIterator.Warnings()) > 0 { log.Info(msgIterator.Warnings()) } if len(participants) == 0 { handleInvalidMessage(msgId, usrId, rawMsgId) } } msgIterator.Close() if count < total { log.Warnf("for some reason, only %d messages have been scan, out of %d", count, total) } else { log.Infof("%d messages scanned", count) } log.Infof("%d invalid messages handled => %d fixed, %d failed", invalidCount, fixedCount, failedCount) } func reInjectRaw(userId, msgId string, msgQueue *nats.Conn) (newMsgId string, err error) { const nats_message_tmpl = "{\"order\":\"process_raw\",\"user_id\": \"%s\", \"message_id\": \"%s\"}" natsMessage := fmt.Sprintf(nats_message_tmpl, userId, msgId) resp, err := msgQueue.Request(lmtpConf.LDAConfig.InTopic, []byte(natsMessage), 10*time.Second) if err != nil { if msgQueue.LastError() != nil { log.WithError(msgQueue.LastError()).Warnf("[EmailBroker] failed to publish inbound request on NATS for user %s", userId) log.Infof("natsMessage: %s\nnatsResponse: %+v\n", natsMessage, resp) return } else { log.WithError(err).Warnf("[EmailBroker] failed to publish inbound request on NATS for user %s", userId) log.Infof("natsMessage: %s\nnatsResponse: %+v\n", natsMessage, resp) return } } nats_ack := new(map[string]interface{}) err = json.Unmarshal(resp.Data, &nats_ack) if err != nil { log.WithError(err).Warnf("[EmailBroker] failed to parse inbound ack on NATS for user %s", userId) log.Infof("natsMessage: %s\nnatsResponse: %+v\n", natsMessage, resp) return } if e, ok := (*nats_ack)["error"]; ok { log.WithError(errors.New(e.(string))).Warnf("[EmailBroker] inbound delivery failed for user %s", userId) log.Infof("natsMessage: %s\nnatsResponse: %+v\n", natsMessage, resp) err = errors.New(e.(string)) return } newMsgId = (*nats_ack)["message_id"].(string) return } ================================================ FILE: src/backend/tools/go.CLI/cmd/gocaliopen/cli_cmds/identitiesMigration.go ================================================ // Copyleft (ɔ) 2018 The Caliopen contributors. // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC // license (AGPL) that can be found in the LICENSE file. package cmd import ( "errors" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/cassandra" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/REST" log "github.com/Sirupsen/logrus" "github.com/gocql/gocql" "github.com/satori/go.uuid" "github.com/spf13/cobra" "time" ) var identitiesMigrationCmd = &cobra.Command{ Use: "identitiesMigration", Short: "manage db and index migration for new model UserIdentity", Long: `command merges LocalIdentities and RemoteIdentities into new UserIdentities, builds lookup tables, then iterates over all messages in db and index to replace identities by user_identities`, Run: identitiesMigration, } func init() { RootCmd.AddCommand(identitiesMigrationCmd) } func identitiesMigration(cmd *cobra.Command, args []string) { // this script needs to read usernames from vault // replace apiv2's credentials with imapworker's credentials apiConf.BackendConfig.Settings.VaultSettings.Username = imapWorkerConf.StoreConfig.VaultConfig.Username apiConf.BackendConfig.Settings.VaultSettings.Password = imapWorkerConf.StoreConfig.VaultConfig.Password apiConf.BackendConfig.Settings.VaultSettings.Url = imapWorkerConf.StoreConfig.VaultConfig.Url var err error //check/get connexions to facilities //store var Store *store.CassandraBackend Store, err = getStoreFacility() if err != nil { log.WithError(err).Fatalf("initialization of %s backend failed", apiConf.BackendName) } defer Store.Close() var Rest *REST.RESTfacility Rest, err = getRESTFacility() if err != nil { log.WithError(err).Fatal("initialization of ReST facility failed") } /* get all users id */ erroneousUsers := []string{} erroneousMessages := []string{} if users, err := Store.Session.Query(`SELECT user_id, shard_id, local_identities FROM user`).Iter().SliceMap(); err != nil { log.WithError(err).Fatal("failed to retrieve user ids") } else { users_total := len(users) users_count := 0 log.Infof("\nFound %d users to work with\n", users_total) for _, user := range users { users_count++ log.Infof("starting migration for user %d/%d", users_count, users_total) var userId UUID userId.UnmarshalBinary(user["user_id"].(gocql.UUID).Bytes()) shardId := user["shard_id"].(string) var localId *UUID var err error // check if a local user_identity has already been created localIds, err := Rest.RetrieveLocalIdentities(userId.String()) if err != nil || len(localIds) == 0 { localId, err = createLocal(user, Store, erroneousUsers, userId, Rest) if err != nil { log.WithError(err).Warnf("createLocal returned error for user %s", userId.String()) } } else { localId = &(localIds[0].Id) } err = createRemotes(Store, erroneousUsers, userId, Rest) if err != nil { log.WithError(err).Warnf("createRemote returned error for user %s", userId.String()) } if localId != nil { err = updateSentMessages(Store, userId, shardId, Rest, erroneousMessages, *localId) if err != nil { log.WithError(err).Warnf("updateSentMessages returned error for user %s", userId.String()) } } } log.Info("\nAll done\n") if len(erroneousUsers) > 0 { log.Warnf("Users with errors : %v", erroneousUsers) } if len(erroneousMessages) > 0 { log.Warnf("Messages with errors : %v", erroneousMessages) } } } func createLocal(user map[string]interface{}, Store *store.CassandraBackend, erroneousUsers []string, userId UUID, Rest *REST.RESTfacility) (localId *UUID, err error) { log.Info("creating UserIdentity for local") local := map[string]interface{}{} identifier := user["local_identities"].([]string) if len(identifier) > 0 { err = Store.Session.Query(`SELECT * FROM local_identity WHERE identifier = ?`, identifier[0]).MapScan(local) if err != nil { log.WithError(err).Warnf("failed to retrieve local identity '%s' for user %s", identifier[0], userId.String()) erroneousUsers = append(erroneousUsers, userId.String()) return } uid := new(UUID) err = uid.UnmarshalBinary(uuid.NewV4().Bytes()) if err != nil { log.WithError(err).Warnf("failed to create uuid for local identity for user %s", userId.String()) erroneousUsers = append(erroneousUsers, userId.String()) return } localUser := UserIdentity{ DisplayName: local["display_name"].(string), Id: *uid, Identifier: local["identifier"].(string), Protocol: "smtp", Status: "active", Type: "local", UserId: userId, } caliopenErr := Rest.CreateUserIdentity(&localUser) if caliopenErr != nil { log.WithError(caliopenErr).Warnf("failed to save local identity %v for user %s", localUser, userId.String()) erroneousUsers = append(erroneousUsers, userId.String()) return nil, caliopenErr.Cause() } return uid, nil } return nil, errors.New("local_identities empty") } func createRemotes(Store *store.CassandraBackend, erroneousUsers []string, userId UUID, Rest *REST.RESTfacility) (err error) { log.Info("creating UserIdentity for remotes") remotes, err := Store.Session.Query(`SELECT * from remote_identity WHERE user_id = ?`, userId.String()).Iter().SliceMap() if err != nil { log.WithError(err).Warnf("failed to retrieve remote identities for user %s", userId.String()) erroneousUsers = append(erroneousUsers, userId.String()) return } for _, remote := range remotes { remoteID := new(UserIdentity) remoteID.NewEmpty() if dn, ok := remote["display_name"].(string); ok { remoteID.DisplayName = dn } if infos, ok := remote["infos"].(map[string]string); ok { remoteID.Infos = make(map[string]string) for k, v := range infos { remoteID.Infos[k] = v } } if lc, ok := remote["last_check"].(time.Time); ok { remoteID.LastCheck = lc } if remote_id, ok := remote["remote_id"].(gocql.UUID); ok { remoteID.Id.UnmarshalBinary(remote_id.Bytes()) } if status, ok := remote["status"].(string); ok { remoteID.Status = status } if t, ok := remote["type"].(string); ok { remoteID.Protocol = t } if userid, ok := remote["user_id"].(gocql.UUID); ok { remoteID.UserId.UnmarshalBinary(userid.Bytes()) } //try to fill-in identifier from username or display name if apiConf.BackendConfig.Settings.UseVault { cred, err := Store.RetrieveCredentials(userId.String(), remoteID.Id.String()) if err != nil { remoteID.Identifier = remoteID.DisplayName } else { if username, ok := cred["username"]; ok { remoteID.Identifier = username } else { remoteID.Identifier = remoteID.DisplayName } } } else { if credentials, ok := remote["credentials"]; ok && credentials != nil { cred := &Credentials{} cred.UnmarshalCQLMap(credentials.(map[string]string)) remoteID.Credentials = cred if username, ok := credentials.(map[string]string)["username"]; ok { remoteID.Identifier = username } else { remoteID.Identifier = remoteID.DisplayName } } else { remoteID.Identifier = remoteID.DisplayName } } remoteID.Type = RemoteIdentity caliopenErr := Rest.CreateUserIdentity(remoteID) if caliopenErr != nil { log.WithError(caliopenErr).Warnf("failed to save remote identity %v for user %s", *remoteID, userId.String()) erroneousUsers = append(erroneousUsers, userId.String()) continue } } return nil } func updateSentMessages(Store *store.CassandraBackend, userId UUID, shardId string, Rest *REST.RESTfacility, erroneousMessages []string, localId UUID) error { /* get all user's messages id from store */ if messages, err := Store.Session.Query(`SELECT message_id, is_received FROM message WHERE user_id = ? ALLOW FILTERING`, userId.String()).Iter().SliceMap(); err != nil { log.WithError(err).Errorf("failed to retrieve messages for user %s", userId.String()) return err } else { log.Infof("iterating over %d messages for user %s", len(messages), userId) for _, msg := range messages { if !msg["is_received"].(bool) { message, err := Rest.GetMessage(&UserInfo{userId.String(), shardId}, msg["message_id"].(gocql.UUID).String()) if err != nil { continue } userIdentities := []UUID{localId} updatedFields := map[string]interface{}{"UserIdentities": userIdentities} message.UserIdentities = userIdentities err = Store.UpdateMessage(message, updatedFields) if err != nil { log.WithError(err).Warnf("failed to update message %s in db", message.Message_id.String()) erroneousMessages = append(erroneousMessages, userId.String()) } } } } return nil } ================================================ FILE: src/backend/tools/go.CLI/cmd/gocaliopen/cli_cmds/root.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package cmd import ( "errors" "fmt" . "github.com/CaliOpen/Caliopen/src/backend/defs/go-objects" "github.com/CaliOpen/Caliopen/src/backend/interfaces/REST/go.server" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/index/elasticsearch" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/cassandra" "github.com/CaliOpen/Caliopen/src/backend/main/go.backends/store/vault" "github.com/CaliOpen/Caliopen/src/backend/main/go.main/facilities/REST" "github.com/CaliOpen/Caliopen/src/backend/protocols/go.imap" "github.com/CaliOpen/Caliopen/src/backend/protocols/go.smtp" log "github.com/Sirupsen/logrus" "github.com/gocql/gocql" "github.com/nats-io/go-nats" "github.com/spf13/cobra" "github.com/spf13/viper" "net/http" "os" ) type CmdConfig struct { rest_api.APIConfig rest_api.IndexConfig rest_api.ProxyConfig } var ( cfgPath string apiCfgFile string lmtpCfgFile string imapWorkerCfgFile string apiConf CmdConfig lmtpConf caliopen_smtp.SMTPConfig imapWorkerConf imap_worker.WorkerConfig // RootCmd represents the base command when called without any subcommands RootCmd = &cobra.Command{ Use: "gocaliopen", Short: "Caliopen CLI to interact with stack", Long: `gocaliopen needs two of Caliopen's config files : caliopen-go-api_dev.yaml and caliopen-go-lmtp_dev.yaml. It loads them from within directory specified with flag --confPath, or if path/filenames are specified with the --apiConf and --lmtpConf. gocaliopen subcommands could interact with - store (Cassandra) - index (Elasticsearch) - message queue (NATS) - cache (Redis) - caliopen REST facility - apiV1 - apiV2 - lmtpd - caliopen notification facility - email broker `, } ) const __version__ = "0.23.0" func init() { cobra.OnInitialize(initConfig) RootCmd.PersistentFlags().StringVar(&cfgPath, "confPath", "", "Path to seek the two mandatory config files: apiv2.yaml and lmtp.yaml") RootCmd.PersistentFlags().StringVar(&apiCfgFile, "apiConf", "", "Caliopen's API config file") RootCmd.PersistentFlags().StringVar(&lmtpCfgFile, "lmtpConf", "", "Caliopen's lmtpd config file") RootCmd.PersistentFlags().StringVar(&imapWorkerCfgFile, "imapWorkerConf", "", "Imap worker config file") } // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { if err := RootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(1) } } // initConfig reads in config files and ENV variables if set. func initConfig() { apiCfg := viper.New() lmtpCfg := viper.New() imapWorkerCfg := viper.New() if cfgPath != "" { // Use path from the flag apiCfg.AddConfigPath(cfgPath) lmtpCfg.AddConfigPath(cfgPath) imapWorkerCfg.AddConfigPath(cfgPath) } else { // set defaults to current dir apiCfg.AddConfigPath(".") lmtpCfg.AddConfigPath(".") imapWorkerCfg.AddConfigPath(".") } if apiCfgFile != "" { // Use config file name and path from the flag. apiCfg.SetConfigFile(apiCfgFile) } else { apiCfg.SetConfigName("apiv2") } if lmtpCfgFile != "" { // Use config file name and path from the flag. lmtpCfg.SetConfigFile(lmtpCfgFile) } else { lmtpCfg.SetConfigName("lmtp") } if imapWorkerCfgFile != "" { // Use config file name and path from the flag. imapWorkerCfg.SetConfigFile(imapWorkerCfgFile) } else { imapWorkerCfg.SetConfigName("imapworker") } // read in environment variables that match apiCfg.AutomaticEnv() lmtpCfg.AutomaticEnv() imapWorkerCfg.AutomaticEnv() if err := apiCfg.ReadInConfig(); err != nil { log.WithError(err).Fatalf("can't load api config file %s", apiCfgFile) } if err := apiCfg.Unmarshal(&apiConf); err != nil { log.WithError(err).Fatalf("can't parse api config file %s", apiCfgFile) } if err := lmtpCfg.ReadInConfig(); err != nil { log.WithError(err).Fatalf("can't load lmtp config file %s", lmtpCfgFile) } if err := lmtpCfg.Unmarshal(&lmtpConf); err != nil { log.WithError(err).Fatalf("can't parse lmtp config file %s", lmtpCfgFile) } if err := imapWorkerCfg.ReadInConfig(); err != nil { log.WithError(err).Warn("can't load imapworker config file %s", imapWorkerCfgFile) } if err := imapWorkerCfg.Unmarshal(&imapWorkerConf); err != nil { log.WithError(err).Warn("can't parse imapworker config file %s", imapWorkerCfgFile) } } // getStoreFacility reads configuration and tries to connect to a store // It returns a handler to make use of store facility // For now, only returns a CassandraBackend func getStoreFacility() (Store *store.CassandraBackend, err error) { switch apiConf.BackendName { case "cassandra": c := store.CassandraConfig{ Hosts: apiConf.BackendConfig.Settings.Hosts, Keyspace: apiConf.BackendConfig.Settings.Keyspace, Consistency: gocql.Consistency(apiConf.BackendConfig.Settings.Consistency), SizeLimit: apiConf.BackendConfig.Settings.SizeLimit, WithObjStore: true, UseVault: apiConf.BackendConfig.Settings.UseVault, HVaultConfig: vault.HVaultConfig{ apiConf.BackendConfig.Settings.VaultSettings.Url, apiConf.BackendConfig.Settings.VaultSettings.Username, apiConf.BackendConfig.Settings.VaultSettings.Password, }, } c.Endpoint = apiConf.BackendConfig.Settings.ObjStoreSettings.Endpoint c.AccessKey = apiConf.BackendConfig.Settings.ObjStoreSettings.AccessKey c.SecretKey = apiConf.BackendConfig.Settings.ObjStoreSettings.SecretKey c.RawMsgBucket = apiConf.BackendConfig.Settings.ObjStoreSettings.Buckets["raw_messages"] c.AttachmentBucket = apiConf.BackendConfig.Settings.ObjStoreSettings.Buckets["temporary_attachments"] c.Location = apiConf.BackendConfig.Settings.ObjStoreSettings.Location //TODO: add a conf file for gocli. if c.UseVault { c.Url = apiConf.BackendConfig.Settings.VaultSettings.Url c.Username = "gocli" c.Password = "gocli_weak_password" } Store, err = store.InitializeCassandraBackend(c) if err != nil { return nil, err } } return } // getIndexFacility reads configuration and tries to connect to an index // It returns a handler to make use of index facility // For now, only returns an ElasticSearchBackend func getIndexFacility() (Index *index.ElasticSearchBackend, err error) { switch apiConf.APIConfig.IndexConfig.IndexName { case "elasticsearch": c := index.ElasticSearchConfig{ Urls: apiConf.APIConfig.IndexConfig.Settings.Hosts, } Index, err = index.InitializeElasticSearchIndex(c) if err != nil { return nil, err } default: return nil, errors.New("unknown index") } return } // getMsgSystemFacility reads configuration and tries to connect to a messages broker // It returns a handler to make use of facility // For now, only returns an Nats conn func getMsgSystemFacility() (MsgSys *nats.Conn, err error) { MsgSys, err = nats.Connect(apiConf.APIConfig.NatsConfig.Url) if err != nil { return nil, err } return } // getCacheFacility reads configuration and tries to connect to a memory cache // It returns a handler to make use of facility // For now, only returns a RedisBackend /* func getCacheFacility() (Cache *cache.RedisBackend, err error) { Cache, err = cache.InitializeRedisBackend(CacheConfig(apiConf.APIConfig.CacheSettings)) if err != nil { return nil, err } return } */ // getAPIConnection pings API1 and API2 connections and returns URLs from configuration file // OK is false if at least one API is not responding func getAPIConnection() (API1, API2 string, OK1, OK2 bool) { API1 = apiConf.ProxyConfig.Routes["/"] API2 = apiConf.ProxyConfig.Routes["/api/v2/"] if _, err := http.Head("http://" + API1); err == nil { OK1 = true } if _, err := http.Head("http://" + API2); err == nil { OK2 = true } return } // getAPI2Facility reads configuration and initializes RESTfacility interface // to expose all its functions func getRESTFacility() (API2 *REST.RESTfacility, err error) { var MsgSys *nats.Conn MsgSys, err = getMsgSystemFacility() if err != nil { return nil, err } caliopenConf := CaliopenConfig{ RESTstoreConfig: RESTstoreConfig{ BackendName: apiConf.APIConfig.BackendConfig.BackendName, Hosts: apiConf.APIConfig.BackendConfig.Settings.Hosts, Keyspace: apiConf.APIConfig.BackendConfig.Settings.Keyspace, Consistency: apiConf.APIConfig.BackendConfig.Settings.Consistency, SizeLimit: apiConf.APIConfig.BackendConfig.Settings.SizeLimit, ObjStoreType: apiConf.APIConfig.BackendConfig.Settings.ObjStoreType, }, RESTindexConfig: RESTIndexConfig{ IndexName: apiConf.APIConfig.IndexConfig.IndexName, Hosts: apiConf.APIConfig.IndexConfig.Settings.Hosts, }, NatsConfig: NatsConfig{ Url: apiConf.APIConfig.NatsConfig.Url, OutSMTP_topic: apiConf.APIConfig.OutSMTP_topic, Contacts_topic: apiConf.APIConfig.Contacts_topic, }, CacheConfig: CacheConfig{ Host: apiConf.APIConfig.CacheSettings.Host, Password: apiConf.APIConfig.CacheSettings.Password, Db: apiConf.APIConfig.CacheSettings.Db, }, NotifierConfig: NotifierConfig{ AdminUsername: apiConf.APIConfig.NotifierConfig.AdminUsername, BaseUrl: apiConf.APIConfig.NotifierConfig.BaseUrl, TemplatesPath: apiConf.APIConfig.NotifierConfig.TemplatesPath, }, } caliopenConf.RESTstoreConfig.Endpoint = apiConf.APIConfig.BackendConfig.Settings.ObjStoreSettings.Endpoint caliopenConf.RESTstoreConfig.AccessKey = apiConf.APIConfig.BackendConfig.Settings.ObjStoreSettings.AccessKey caliopenConf.RESTstoreConfig.SecretKey = apiConf.APIConfig.BackendConfig.Settings.ObjStoreSettings.SecretKey caliopenConf.RESTstoreConfig.Location = apiConf.APIConfig.BackendConfig.Settings.ObjStoreSettings.Location caliopenConf.RESTstoreConfig.Buckets = apiConf.APIConfig.BackendConfig.Settings.ObjStoreSettings.Buckets API2 = REST.NewRESTfacility(caliopenConf, MsgSys) return API2, nil } func getNotificationsFacility() { /*TODO*/ } func getLMTPFacility() { /*TODO*/ } ================================================ FILE: src/backend/tools/go.CLI/cmd/gocaliopen/main.go ================================================ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ /* * // Copyleft (ɔ) 2018 The Caliopen contributors. * // Use of this source code is governed by a GNU AFFERO GENERAL PUBLIC * // license (AGPL) that can be found in the LICENSE file. */ package main import "github.com/CaliOpen/Caliopen/src/backend/tools/go.CLI/cmd/gocaliopen/cli_cmds" func main() { cmd.Execute() } ================================================ FILE: src/backend/tools/py.CLI/CHANGES.rst ================================================ 0.0.1 ----- - Initial version ================================================ FILE: src/backend/tools/py.CLI/README.rst ================================================ caliopen.cli ============ Caliopen Command Line Interface Simple tool to provide some administration commands for caliopen project. Define a ``caliopen`` command in your shell path. # Usage ## Copy the sample caliopen.yaml.template file to roll your own configuration parameters:: cp caliopen.yaml.template caliopen.yaml ## Setup the storage database:: caliopen -f caliopen.yaml setup ## Create an admin :: caliopen -f caliopen.yaml create_user -e admin -p password ## Create a user:: caliopen create_user --help caliopen -f caliopen.yaml create_user -e imported@email -p password -g given_name -f family_name ## Import a mailbox :: caliopen import --help caliopen -f caliopen.yaml import -p ~/path_to_maildir -e imported@email -f maildir ## Dump a model :: caliopen -f caliopen.yaml dump_model -m model_name -o output_path ## Import vcard :: Refer to [import vcard](../doc/for-developers/vcard_doc.md) ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/__init__.py ================================================ # -*- coding: utf-8 -*- __version__ = '0.27.0' try: import pkg_resources pkg_resources.declare_namespace(__name__) except ImportError: import pkgutil __path__ = pkgutil.extend_path(__path__, __name__) ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/cli.py ================================================ #!/usr/bin/env python """ Command Line Interface (CLI) for caliopen project """ import sys import argparse import logging from caliopen_storage.config import Configuration from caliopen_storage.helpers.connection import connect_storage from caliopen_cli.commands import (shell, import_email, setup, create_user, import_vcard, dump_model, dump_indexes, inject_email, basic_compute, migrate_index, import_reserved_names, resync_index, resync_shard_index, copy_model) logging.basicConfig(level=logging.INFO) def main(args=sys.argv): parser = argparse.ArgumentParser() parser.add_argument('-f', dest='conffile', default='development.ini') subparsers = parser.add_subparsers(title="action") sp_import = subparsers.add_parser('import', help='import existing mailbox') sp_import.set_defaults(func=import_email) sp_import.add_argument('-f', dest='format', choices=['mbox', 'maildir'], default='mbox') sp_import.add_argument('-p', dest='import_path') sp_import.add_argument('-e', dest='email') sp_import.add_argument('--contact-probability', dest='contact_probability', default=0.8) sp_import.add_argument('-t', dest='to') sp_import_vcard = subparsers.add_parser('import_vcard', help='import vcard') sp_import_vcard.set_defaults(func=import_vcard) sp_import_vcard.add_argument('-u', dest='username', help='username') sp_import_vcard.add_argument('-d', dest='directory', help='directory') sp_import_vcard.add_argument('-f', dest='file_vcard', help='file') sp_setup = subparsers.add_parser('setup', help='initialize the storage engine') sp_setup.set_defaults(func=setup) sp_create_user = subparsers.add_parser('create_user', help='Create a new user') sp_create_user.set_defaults(func=create_user) sp_create_user.add_argument('-e', dest='email', help='user email') sp_create_user.add_argument('-p', dest='password', help='password') sp_create_user.add_argument('-g', dest='given_name', help='user given name') sp_create_user.add_argument('-f', dest='family_name', help='user family name') sp_shell = subparsers.add_parser('shell') sp_shell.set_defaults(func=shell) sp_dump = subparsers.add_parser('dump') sp_dump.set_defaults(func=dump_model) sp_dump.add_argument('-m', dest='model', help='model to dump') sp_dump.add_argument('-o', dest='output_path', help='output path') sp_copy = subparsers.add_parser('copy') sp_copy.set_defaults(func=copy_model) sp_copy.add_argument('-m', dest='model', help='model to dump') sp_copy.add_argument('--where', dest='where', help='where condition') sp_copy.add_argument('--fetch-size', dest='fetch_size', default=100) sp_dump_index = subparsers.add_parser('dump_index') sp_dump_index.set_defaults(func=dump_indexes) sp_dump_index.add_argument('-o', dest='output_path', help='output path') sp_migrate_index = subparsers.add_parser('migrate_index') sp_migrate_index.set_defaults(func=migrate_index) sp_migrate_index.add_argument('-s', dest='input_script', help='python script to execute on index') sp_inject = subparsers.add_parser('inject') sp_inject.set_defaults(func=inject_email) sp_inject.add_argument('-e', dest='email') sp_inject.add_argument('-r', dest='recipient') sp_compute = subparsers.add_parser('compute', help='Launch basic compute') sp_compute.set_defaults(func=basic_compute) sp_compute.add_argument('-u', dest='username', help='username') sp_compute.add_argument('-j', dest='job', help='job name') sp_reserved = subparsers.add_parser('reserved_names', help='Import reserved names list') sp_reserved.set_defaults(func=import_reserved_names) sp_reserved.add_argument('-i', dest='input_file', help='csv file') sp_resync = subparsers.add_parser('resync_index', help='Resync index for an user') sp_resync.set_defaults(func=resync_index) sp_resync.add_argument('-u', dest='user_name', help='User name') sp_resync.add_argument('-i', dest='user_id', help='User uuid') sp_resync.add_argument('--delete', dest='delete', action='store_true') sp_resync = subparsers.add_parser('resync_shard', help='Resync shard index') sp_resync.set_defaults(func=resync_shard_index) sp_resync.add_argument('-s', dest='shard_id', help='Shard id') sp_resync.add_argument('-o', dest='old_shard_id', help='Old shard id') kwargs = parser.parse_args(args[1:]) kwargs = vars(kwargs) config_uri = kwargs.pop('conffile') func = kwargs.pop('func') Configuration.load(config_uri, 'global') connect_storage() func(**kwargs) if __name__ == '__main__': main() ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/__init__.py ================================================ from .shell import shell from .setup import setup from .create_user import create_user from .import_email import import_email from .inject_email import inject_email from .import_vcard import import_vcard from .dump_model import dump_model from .dump_indexes_mappings import dump_indexes from .migrate_index import migrate_index from .compute import basic_compute from .reserved_names import import_reserved_names from .resync_index import resync_index from .resync_shard_index import resync_shard_index from .copy_model import copy_model ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/compute.py ================================================ """Launch of nasic compute on caliopen platform.""" from __future__ import absolute_import, print_function, unicode_literals import logging log = logging.getLogger(__name__) def basic_compute(username, job, ** kwargs): """Import emails for an user.""" from caliopen_main.user.core import User from caliopen_main.contact.objects import Contact from caliopen_pi.qualifiers import ContactMessageQualifier user = User.by_name(username) qualifier = ContactMessageQualifier(user) contacts = Contact.list_db(user.user_id) if job == 'contact_privacy': for contact in contacts: log.info('Processing contact {0}'.format(contact.contact_id)) qualifier.process(contact) ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/copy_model.py ================================================ import sys from cassandra.cluster import Cluster from cassandra.query import SimpleStatement from cassandra.query import dict_factory from caliopen_storage.config import Configuration def copy_model(**kwargs): conf = Configuration('global').configuration cluster_source = Cluster(conf['old_cassandra']['hosts']) source = cluster_source.connect(conf['old_cassandra']['keyspace']) source.row_factory = dict_factory cluster_dest = Cluster(conf['cassandra']['hosts']) dest = cluster_dest.connect(conf['cassandra']['keyspace']) table = kwargs['model'].lower() fetch_size = kwargs.get('fetch_size', 100) query = "SELECT * FROM {0}".format(table) if 'where' in kwargs and kwargs['where']: query = "{0} WHERE {1} ALLOW FILTERING".format(query, kwargs['where']) statement = SimpleStatement(query, fetch_size=fetch_size) insert_query = "INSERT INTO {0} ({1}) VALUES ({2})" cpt = 0 insert = None for row in source.execute(statement): if cpt == 0: columns = ['"{}"'.format(x) for x in row.keys()] binds = ['?' for x in range(0, len(columns))] insert_str = insert_query.format(table, ','.join(columns), ','.join(binds)) insert = dest.prepare(insert_str) bound = insert.bind(row.values()) dest.execute(bound) cpt += 1 print('Copy of {} records from {}'.format(cpt, table)) return cpt ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/create_user.py ================================================ """Create a user with a password in a Calipen instance.""" from __future__ import absolute_import, print_function, unicode_literals import logging from caliopen_storage.config import Configuration log = logging.getLogger(__name__) def create_user(**kwargs): """Create user in Caliopen instance.""" from caliopen_main.user.core import User from caliopen_main.user.parameters import NewUser from caliopen_main.contact.parameters import NewContact from caliopen_main.contact.parameters import NewEmail # Fill core registry from caliopen_main.message.objects.message import Message param = NewUser() param.name = kwargs['email'] if '@' in param.name: username, domain = param.name.split('@') param.name = username # Monkey patch configuration local_domain with provided value conf = Configuration('global').configuration conf['default_domain'] = domain param.password = kwargs['password'] param.recovery_email = u'{}@recovery-caliopen.local'.format(param.name) contact = NewContact() contact.given_name = kwargs.get('given_name') contact.family_name = kwargs.get('family_name') email = NewEmail() email.address = param.recovery_email contact.emails = [email] param.contact = contact user = User.create(param) log.info('User %s (%s) created' % (user.user_id, user.name)) ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/dump_indexes_mappings.py ================================================ import json from caliopen_storage.helpers.json import JSONEncoder def dump_indexes(**kwargs): # Discover base core classes from caliopen_main.user.core import User from caliopen_main.contact.objects.contact import Contact from caliopen_main.message.objects.message import Message from caliopen_main.common.objects.tag import ResourceTag from caliopen_storage.core import core_registry _exports = { 'contact': ['Contact'], 'message': ['Message'], } for keys in _exports: for obj in _exports[keys]: kls = core_registry.get(obj) if not kls: raise Exception('core class %s not found in registry' % obj) output_file = '%s/%s.json' % (kwargs["output_path"], obj.lower()) dump_index_mapping(kls._index_class, output_file) def dump_index_mapping(kls, output_file): """Output the json definition class.""" m = kls.build_mapping().to_dict() with open(output_file, 'w') as f: f.write(json.dumps(m, cls=JSONEncoder, indent=4, sort_keys=True)) ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/dump_model.py ================================================ from caliopen_storage.helpers.json import json, JSONEncoder def dump_model(model, output_path, **kwargs): # Discover base core classes from caliopen_main.user.core import User from caliopen_storage.core import core_registry _exports = { 'contact': ['Contact', 'Organization', 'PostalAddress', 'Email', 'IM', 'Ohone', 'PublicKey', 'SocialIdentity'], 'message': ['Message', 'Thread'], 'user': ['User', 'UserTag', 'FilterRule'], } for obj in _exports[model]: kls = core_registry.get(obj) if not kls: raise Exception('core class %s not found in registry' % obj) output_file = '%s/%s.json' % (output_path, obj.lower()) dump_model_class(kls, output_file) def dump_model_class(kls, output_file): """Dump a model class into output file.""" records = kls._model_class.objects.all() data = [] for rec in records: d = {} for k in rec._columns.keys(): attr = getattr(rec, k) if hasattr(attr, 'to_python'): d[k] = attr.to_python() else: d[k] = attr data.append(d) with open(output_file, 'w') as f: f.write(json.dumps(data, cls=JSONEncoder, indent=4, sort_keys=True)) ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/import_email.py ================================================ """ This script parse mail from a mbox or maildir format and import them. User must be created before import """ from __future__ import absolute_import, print_function, unicode_literals import os import re from random import random import logging from email import message_from_string, message_from_file from mailbox import mbox, Maildir from caliopen_storage.exception import NotFound from caliopen_main.common.errors import DuplicateMessage log = logging.getLogger(__name__) def import_email(email, import_path, format, contact_probability, **kwargs): """Import emails for an user.""" from caliopen_main.user.core import User from caliopen_main.contact.core import Contact, ContactLookup from caliopen_main.message.parsers.mail import MailMessage from caliopen_main.contact.parameters import NewContact, NewEmail from caliopen_nats.delivery import UserMailDelivery from caliopen_main.message.core import RawMessage from caliopen_storage.config import Configuration max_size = int(Configuration("global").get("object_store.db_size_limit")) if 'to' in kwargs and kwargs['to']: dest_email = kwargs['to'] else: dest_email = email if format == 'maildir': if dest_email != email: raise Exception('Cannot change To email using maildir format') emails = Maildir(import_path, factory=message_from_file) mode = 'maildir' else: if os.path.isdir(import_path): mode = 'mbox_directory' emails = {} files = [f for f in os.listdir(import_path) if os.path.isfile(os.path.join(import_path, f))] for f in files: try: log.debug('Importing mail from file {}'.format(f)) with open('%s/%s' % (import_path, f)) as fh: data = fh.read() data = re.sub('^To: (.*)', 'To: %s' % dest_email, data, flags=re.MULTILINE) emails[f] = message_from_string(data) except Exception as exc: log.error('Error importing email {}'.format(exc)) else: mode = 'mbox' emails = mbox(import_path) user = User.by_local_identifier(dest_email, 'email') log.info("Processing mode %s" % mode) for key, data in emails.iteritems(): # Prevent creating message too large to fit in db. # (should use inject cmd for large messages) size = len(data.as_string()) if size > max_size: log.warn("Message too large to fit into db. \ Please, use 'inject' cmd for importing large emails.") continue raw = RawMessage.create(data.as_string()) log.debug('Created raw message {}'.format(raw.raw_msg_id)) message = MailMessage(data.as_string()) dice = random() if dice <= contact_probability: for participant in message.participants: try: ContactLookup.get(user, participant.address) except NotFound: log.info('Creating contact %s' % participant.address) name, domain = participant.address.split('@') contact_param = NewContact() contact_param.family_name = name if participant.address: e_mail = NewEmail() e_mail.address = participant.address contact_param.emails = [e_mail] Contact.create(user, contact_param) else: log.info('No contact associated to raw {} '.format(raw.raw_msg_id)) processor = UserMailDelivery(user, user.local_identities[ 0]) # assume one local identity try: obj_message = processor.process_raw(raw.raw_msg_id) except Exception as exc: if isinstance(exc, DuplicateMessage): log.info('duplicate message {}, not imported'.format( raw.raw_msg_id)) else: log.exception(exc) else: log.info('Created message {}'.format(obj_message.message_id)) ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/import_vcard.py ================================================ import logging import os log = logging.getLogger(__name__) def import_vcard(username, directory, file_vcard, **kwargs): from caliopen_main.contact.core import Contact as CoreContact from caliopen_main.user.core.user import User as CoreUser from caliopen_main.contact.parsers import VcardParser new_contacts = [] if directory: files = [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f))] for f in files: ext = f.split('.')[-1] if ext == 'vcard' or ext == 'vcf': file = '{directory}/{file}'.format(directory=directory, file=f) parser = VcardParser(file) new_contacts.extend(parser.parse()) else: log.warn("Not valid file extension for vcard %s" % f) if file_vcard: parser = VcardParser(file_vcard) new_contacts = parser.parse() user = CoreUser.by_name(username) for contact in new_contacts: CoreContact.create(user, contact.contact) ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/inject_email.py ================================================ """ This script parse for a local user email (1st arg) and a text file (2nd arg). Text file SHOULD be an RFC5322 email, with or without embedded attachment. It will be streamed to lmtpd (aka 'broker' in docker stack). User must be created before import """ from __future__ import absolute_import, print_function, unicode_literals import smtplib import logging log = logging.getLogger(__name__) def inject_email(recipient, email, **kwargs): """Inject an email for an user.""" with open(email) as f: conn = smtplib.SMTP('localhost', 2525) try: conn.sendmail("inject_email@py.cli.caliopen", [recipient], str(f.read())) except Exception as exc: log.exception(exc) conn.quit() ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/migrate_index.py ================================================ # -*- coding: utf-8 -*- # migrate_index will load the script at the given path # this script must implement a class named "IndexMigrator" # with a method "run(elasticsearch_client)" from __future__ import absolute_import, print_function, unicode_literals import logging import imp import os from elasticsearch import Elasticsearch from caliopen_storage.config import Configuration log = logging.getLogger(__name__) def migrate_index(**kwargs): raise Exception("This script cannot be execute in this current state; it's based on versionned indices which is not used anymore, the version was in the name. So it breaks the migration mechinism. You can use sync_indices command instead to sync settings & mapping, it's almost like a migration except it uses latest version of the mapping.") # TODO eventually reintroduce version but in `_meta` in mapping configuration. # Also remove url in params since it's already set in client and # any way to have an interface for Migrators? # Migrator = load_from_file(kwargs["input_script"]) # if Migrator: # url = Configuration('global').get('elasticsearch.url') # mappings_version = Configuration('global').get( # 'elasticsearch.mappings_version') # if url and mappings_version: # client = Elasticsearch(url) # migration = Migrator(client=client, # mappings_version=mappings_version, # url=url) # migration.run() def load_from_file(filepath): c = None expected_class = 'IndexMigrator' mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1]) if file_ext.lower() == '.py': py_mod = imp.load_source(mod_name, filepath) if hasattr(py_mod, expected_class): c = getattr(py_mod, expected_class) return c ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/reserved_names.py ================================================ """Launch of nasic compute on caliopen platform.""" from __future__ import absolute_import, print_function, unicode_literals import logging log = logging.getLogger(__name__) def import_reserved_names(input_file, ** kwargs): """Import emails for an user.""" from caliopen_main.user.core import ReservedName cpt = 0 with open(input_file) as f: for name in f.read().decode('utf8').split('\n'): if not name.startswith('#'): ReservedName.create(name=name) cpt += 1 log.info('Created {} reserved names'.format(cpt)) ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/resync_index.py ================================================ # -*- coding: utf-8 -*- # Resync data index for an user # from __future__ import absolute_import, print_function, unicode_literals import logging import sys import uuid from elasticsearch import Elasticsearch from caliopen_storage.config import Configuration log = logging.getLogger(__name__) log.setLevel(logging.INFO) def clean_index_user(client, user): """Delete old data belong to an user from index.""" query = {'query': {'term': {'user_id': user.user_id}}} r1 = client.delete_by_query(index=user.shard_id, doc_type='indexed_message', body=query) r2 = client.delete_by_query(index=user.shard_id, doc_type='indexed_contact', body=query) return r1['deleted'], r2['deleted'] def resync_index(**kwargs): """Resync an index for an user.""" from caliopen_main.user.core import User, allocate_user_shard from caliopen_main.user.core.setups import setup_index from caliopen_main.contact.store import Contact from caliopen_main.contact.objects import Contact as ContactObject from caliopen_main.message.store import Message from caliopen_main.message.objects import Message as MessageObject if 'user_name' in kwargs and kwargs['user_name']: user = User.by_name(kwargs['user_name']) elif 'user_id' in kwargs and kwargs['user_id']: user = User.get(kwargs['user_id']) else: print('Need user_name or user_id parameter') sys.exit(1) es_url = Configuration('global').get('elasticsearch.url') es_client = Elasticsearch(es_url) if 'delete' in kwargs and kwargs['delete']: del_msg, del_con = clean_index_user(es_client, user) log.info('Delete of {0} old contacts and {1} old messages'. format(del_con, del_msg)) user_id = uuid.UUID(user.user_id) shard_id = allocate_user_shard(user_id) if user.shard_id != shard_id: log.warn('Reallocate user index shard from {} to {}'. format(user.shard_id, shard_id)) # XXX fixme. attribute should be set without using model user.model.shard_id = shard_id user.save() setup_index(user) cpt_contact = 0 contacts = Contact.filter(user_id=user.user_id) for contact in contacts: log.debug('Reindex contact %r' % contact.contact_id) obj = ContactObject(user, contact_id=contact.contact_id) obj.get_db() obj.unmarshall_db() obj.create_index() cpt_contact += 1 cpt_message = 0 skip_message = 0 messages = Message.filter(user_id=user.user_id).timeout(None). \ allow_filtering() for message in messages: log.debug('Reindex message %r' % message.message_id) try: obj = MessageObject(user, message_id=message.message_id) obj.get_db() obj.unmarshall_db() obj.create_index() cpt_message += 1 except Exception as exc: skip_message += 1 log.exception(exc) log.info('Sync of {0} contacts, {1} messages. Skipped {2} messages.'. format(cpt_contact, cpt_message, skip_message)) log.info('Create index alias %r' % user.user_id) try: es_client.indices.put_alias(index=shard_id, name=user.user_id) except Exception as exc: log.exception('Error during alias creation : {}'.format(exc)) raise exc ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/resync_shard_index.py ================================================ # -*- coding: utf-8 -*- # resync index. This command will delete the current index for an user # And rebuild it entirely using cassandra data # from __future__ import absolute_import, print_function, unicode_literals import logging import sys from caliopen_storage.config import Configuration from elasticsearch import Elasticsearch log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) def resync_user(user): """Resync data for an user into its index shard.""" from caliopen_main.contact.store import Contact from caliopen_main.contact.objects import Contact as ContactObject from caliopen_main.message.store import Message from caliopen_main.message.objects import Message as MessageObject log.info('Sync user {0} into shard {1}'.format(user.user_id, user.shard_id)) client = Elasticsearch(Configuration('global').get('elasticsearch.url')) body = {'filter': {'term': {'user_id': user.user_id}}} # if not client.indices.exists_alias(user.user_id): # log.info('Creating alias {} for index {}'.format(user.user_id, user.shard_id)) client.indices.put_alias(user.shard_id, user.user_id, body=body) contacts = Contact.filter(user_id=user.user_id).timeout(None) for contact in contacts: log.debug('Reindex contact %r' % contact.contact_id) obj = ContactObject(user, contact_id=contact.contact_id) obj.get_db() obj.unmarshall_db() obj.create_index() messages = Message.filter(user_id=user.user_id). \ allow_filtering().timeout(None) for message in messages: try: log.debug('Reindex message %r' % message.message_id) obj = MessageObject(user, message_id=message.message_id) obj.get_db() obj.unmarshall_db() obj.create_index() except Exception as exc: log.error('{}: reindexing message {} for user {}. Continue'.format(type(exc).__name__, message.message_id, user.user_id)) raise def resync_shard_index(**kwargs): """Resync all index of a shard.""" from caliopen_main.user.core import User from caliopen_main.user.core.setups import setup_shard_index shard_id = kwargs['shard_id'] old_shard_id = kwargs.get('old_shard_id') if not old_shard_id: old_shard_id = shard_id shards = Configuration('global').get('elasticsearch.shards') if shard_id not in shards: log.error('Invalid shard {0}'.format(shard_id)) sys.exit(1) # Recreate index and mappings setup_shard_index(shard_id) users = User._model_class.all() cpt = 0 skip = 0 for user in users: if user.shard_id not in (old_shard_id, shard_id): continue if user.shard_id != shard_id: user.shard_id = shard_id user.save() try: resync_user(user) cpt += 1 except Exception as exc: skip += 1 log.exception(exc) log.info('Sync {0} users into shards. Skipped {1} users'.format(cpt, skip)) ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/setup.py ================================================ """Setup backend.""" import logging from .setup_storage import setup_storage from .setup_notifications_ttls import setup_notifications_ttls log = logging.getLogger(__name__) def setup(): """Setup backend, storage and configuration.""" log.info('Setup storage') setup_storage() log.info('Setup notification ttls') setup_notifications_ttls() ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/setup_notifications_ttls.py ================================================ """Create a user with a password in a Calipen instance.""" from __future__ import absolute_import, print_function, unicode_literals import logging log = logging.getLogger(__name__) def setup_notifications_ttls(): """Fill up table `notification_ttl` with default ttls in seconds""" from caliopen_main.notification.core import NotificationTtl default_ttls = { "short-lived": 60, # a minute "mid-lived": 3600, # an hour "long-lived": 43200, # 12 hours "short-term": 86400, # a day "mid-term": 172800, # 2 days "long-term": 1728000, # 10 days "forever": 0 } for k, v in default_ttls.items(): NotificationTtl.create(ttl_code=k, ttl_duration=v) log.info('{} default ttls have been set'.format(len(default_ttls))) ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/setup_storage.py ================================================ """Setup storage backend.""" import logging from caliopen_storage.config import Configuration log = logging.getLogger(__name__) def setup_storage(settings=None): """Create cassandra models.""" from caliopen_storage.core import core_registry # Make discovery happen from caliopen_main.user.core import User from caliopen_main.user.core import (UserIdentity, IdentityLookup, IdentityTypeLookup) from caliopen_main.contact.objects.contact import Contact from caliopen_main.message.objects.message import Message from caliopen_main.common.objects.tag import ResourceTag from caliopen_main.device.core import Device from caliopen_main.notification.core import Notification, NotificationTtl from caliopen_main.protocol.core import Provider from cassandra.cqlengine.management import sync_table, \ create_keyspace_simple keyspace = Configuration('global').get('cassandra.keyspace') if not keyspace: raise Exception('Configuration missing for cassandra keyspace') # XXX tofix : define strategy and replication_factor in configuration create_keyspace_simple(keyspace, 1) for name, kls in core_registry.items(): log.info('Creating cassandra model %s' % name) if hasattr(kls._model_class, 'pk'): # XXX find a better way to detect model from udt sync_table(kls._model_class) ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/commands/shell.py ================================================ """ Caliopen Shell using Ipython if available. """ def shell(**kwargs): try: from IPython import embed from traitlets.config.loader import Config cfg = Config() cfg.InteractiveShellEmbed.confirm_exit = False embed(config=cfg, banner1="Caliopen Shell") except ImportError: # try ~IPython-0.10 API try: from IPython.Shell import IPShellEmbed as embed ipshell = embed(banner="Caliopen Shell") ipshell() except ImportError: import code code.interact("Caliopen Shell", local=locals()) ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/utils/__init__.py ================================================ ================================================ FILE: src/backend/tools/py.CLI/caliopen_cli/utils/user_token.py ================================================ import logging import redis import json log = logging.getLogger(__name__) class UserToken(object): """Utility class to deal with user token session in cache.""" _token_prefix = 'tokens::' def __init__(self, redis_host, redis_port=6379): self.client = redis.Redis(redis_host, redis_port) def parse_session(self, session): if '-' not in session: return session, None user_id, device_id = session.split('-') return user_id, device_id def list_user_sessions(self, user_id): return self.client.keys('{}{}-*'.format(self._token_prefix, user_id)) def get_token(self, token): infos = self.client.get(token) if infos: return json.loads(infos) return {} def delete_token(self, token): return self.client.delete(token) def get_user_status(self, user_id): sessions = self.list_user_sessions(user_id) for session in sessions: token = self.get_token(session) log.info('User session {} status is {}'. format(session, token.get('user_status'))) def _set_user_status(self, user_id, status): sessions = self.list_user_sessions(user_id) for session in sessions: token = self.get_token(session) if token.get('user_status') != status: log.info('Updating user session {} status '.format(session)) token['user_status'] = status self.client.set(session, json.dumps(token)) return True def user_set_maintenance(self, user_id): return self._set_user_status(user_id, 'maintenance') def user_unset_maintenance(self, user_id): return self._set_user_status(user_id, 'active') ================================================ FILE: src/backend/tools/py.CLI/requirements.deps ================================================ caliopen_pi caliopen_nats ================================================ FILE: src/backend/tools/py.CLI/setup.cfg ================================================ [nosetests] match = ^test nocapture = 1 cover-package = caliop with-coverage = 1 cover-erase = 1 ================================================ FILE: src/backend/tools/py.CLI/setup.py ================================================ import os import re from setuptools import setup, find_packages here = os.path.abspath(os.path.dirname(__file__)) README = open(os.path.join(here, 'README.rst')).read() CHANGES = open(os.path.join(here, 'CHANGES.rst')).read() name = "caliopen_cli" with open(os.path.join(*([here] + name.split('.') + ['__init__.py']))) as v_file: version = re.compile(r".*__version__ = '(.*?)'", re.S).match(v_file.read()).group(1) requires = [ ] if (os.path.isfile('./requirements.deps')): with open('./requirements.deps') as f_deps: requires.extend(f_deps.read().split('\n')) setup(name=name, namespace_packages=[name], version=version, description='Caliopen Command Line Interface`', long_description=README + '\n\n' + CHANGES, classifiers=[ "Programming Language :: Python", "Topic :: Shell", ], author='Caliopen Contributors', author_email='', url='https://github.com/Caliopen/caliopen.cli', keywords='caliopen cli', packages=find_packages(), include_package_data=True, zip_safe=False, install_requires=requires, tests_require=requires, test_suite="caliopen.cli.tests", entry_points={ 'console_scripts': 'caliopen = caliopen_cli.cli:main', }) ================================================ FILE: src/backend/tools/py.ML/CHANGES.rst ================================================ ================================================ FILE: src/backend/tools/py.ML/README.rst ================================================ ================================================ FILE: src/backend/tools/py.ML/caliopen_climl/__init__.py ================================================ __version__ = '0.23.0' try: import pkg_resources pkg_resources.declare_namespace(__name__) except ImportError: import pkgutil __path__ = pkgutil.extend_path(__path__, __name__) ================================================ FILE: src/backend/tools/py.ML/caliopen_climl/cli.py ================================================ """Command Line Interface (CLI) for caliopen machine learning tasks.""" import click from caliopen_tag.models_manager import ModelManager, ESDataManager from caliopen_data import save_file class Config(object): """Delay configuration load and storage connection.""" def __init__(self, filename): from caliopen_storage.config import Configuration from caliopen_storage.helpers.connection import connect_storage self.conf = Configuration.load(filename, 'global').configuration connect_storage() @click.group() @click.option('--config', 'config') @click.pass_context def cli(ctx, config): """Entry point.""" ctx.obj = Config(config).conf @cli.command() @click.argument('model') @click.option('--index', default='all') @click.option('--output') @click.pass_obj def train(config, model, index, output): """Train a model command.""" click.echo('Will train model {0} with index {1}'. format(model, index)) if model == 'tagger': provider = ESDataManager(config) provider.prepare(provider.get_query(), index=None, doc_type='indexed_message') manager = ModelManager(provider) result_file = manager.get_new_model(output) save_file(config, result_file, model) else: click.echo('Unknow model {0}'.format(model)) ================================================ FILE: src/backend/tools/py.ML/requirements.deps ================================================ caliopen_tag ================================================ FILE: src/backend/tools/py.ML/setup.cfg ================================================ [nosetests] match = ^test nocapture = 1 cover-package = caliopen with-coverage = 1 cover-erase = 1 ================================================ FILE: src/backend/tools/py.ML/setup.py ================================================ import os import re from setuptools import setup, find_packages here = os.path.abspath(os.path.dirname(__file__)) README = open(os.path.join(here, 'README.rst')).read() CHANGES = open(os.path.join(here, 'CHANGES.rst')).read() name = "caliopen_climl" version_file = os.path.join(*([here] + name.split('.') + ['__init__.py'])) with open(version_file) as v_file: comp = re.compile(r".*__version__ = '(.*?)'", re.S) version = comp.match(v_file.read()).group(1) requires = [ 'click' ] if (os.path.isfile('./requirements.deps')): with open('./requirements.deps') as f_deps: requires.extend(f_deps.read().split('\n')) setup(name=name, namespace_packages=[name], version=version, description='Caliopen CLI interface for Machine Learning tasks`', long_description=README + '\n\n' + CHANGES, classifiers=["Programming Language :: Python", "Topic :: Shell"], author='Caliopen Contributors', author_email='', url='https://github.com/Caliopen', keywords='caliopen machine learning CLI', packages=find_packages(), include_package_data=True, zip_safe=False, install_requires=requires, tests_require=requires, test_suite="caliopen_climl.cli.tests", entry_points={ 'console_scripts': 'caliopml = caliopen_climl.cli:cli', }) ================================================ FILE: src/backend/tools/py.doc/CHANGES.rst ================================================ 0.0.2 ----- - Initial version ================================================ FILE: src/backend/tools/py.doc/README.rst ================================================ Caliopen swagger automatic API plugin package ============================================= This package permit to include in a caliopen development platform, the swagger-ui tool for documentation and also interaction with the ReST API. When installed and configured into your pyramid.includes section as `caliopen_api_doc`, if you are using a different configuration file for pyramid than the provided one for development or docker environment, you then can use it under localhost:6543/api-ui/# ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/__init__.py ================================================ # -*- coding: utf-8 -*- __version__ = '0.0.2' from .config import includeme ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/config.py ================================================ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals import logging log = logging.getLogger(__name__) def includeme(config): """Configure API to serve static documentation files.""" log.info('Loading api doc module') settings = config.registry.settings swagger_dir = settings.get('pyramid_swagger.schema_directory') if not swagger_dir: log.warn('No configured swagger schema directory found') else: log.info('Will load swagger.json from {}'.format(swagger_dir)) config.add_static_view('doc/api', swagger_dir, cache_max_age=3600) # the api swagger-ui is within folder /devtools/swagger-ui config.add_static_view('api-ui', 'caliopen_api_doc:swagger-ui/', cache_max_age=3600) # the Single Source of Truth config.add_static_view('defs', 'caliopen_api_doc:../../defs/', cache_max_age=3600) ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/css/print.css ================================================ /* Original style from softwaremaniacs.org (c) Ivan Sagalaev */ .swagger-section pre code { display: block; padding: 0.5em; background: #F0F0F0; } .swagger-section pre code, .swagger-section pre .subst, .swagger-section pre .tag .title, .swagger-section pre .lisp .title, .swagger-section pre .clojure .built_in, .swagger-section pre .nginx .title { color: black; } .swagger-section pre .string, .swagger-section pre .title, .swagger-section pre .constant, .swagger-section pre .parent, .swagger-section pre .tag .value, .swagger-section pre .rules .value, .swagger-section pre .rules .value .number, .swagger-section pre .preprocessor, .swagger-section pre .ruby .symbol, .swagger-section pre .ruby .symbol .string, .swagger-section pre .aggregate, .swagger-section pre .template_tag, .swagger-section pre .django .variable, .swagger-section pre .smalltalk .class, .swagger-section pre .addition, .swagger-section pre .flow, .swagger-section pre .stream, .swagger-section pre .bash .variable, .swagger-section pre .apache .tag, .swagger-section pre .apache .cbracket, .swagger-section pre .tex .command, .swagger-section pre .tex .special, .swagger-section pre .erlang_repl .function_or_atom, .swagger-section pre .markdown .header { color: #800; } .swagger-section pre .comment, .swagger-section pre .annotation, .swagger-section pre .template_comment, .swagger-section pre .diff .header, .swagger-section pre .chunk, .swagger-section pre .markdown .blockquote { color: #888; } .swagger-section pre .number, .swagger-section pre .date, .swagger-section pre .regexp, .swagger-section pre .literal, .swagger-section pre .smalltalk .symbol, .swagger-section pre .smalltalk .char, .swagger-section pre .go .constant, .swagger-section pre .change, .swagger-section pre .markdown .bullet, .swagger-section pre .markdown .link_url { color: #080; } .swagger-section pre .label, .swagger-section pre .javadoc, .swagger-section pre .ruby .string, .swagger-section pre .decorator, .swagger-section pre .filter .argument, .swagger-section pre .localvars, .swagger-section pre .array, .swagger-section pre .attr_selector, .swagger-section pre .important, .swagger-section pre .pseudo, .swagger-section pre .pi, .swagger-section pre .doctype, .swagger-section pre .deletion, .swagger-section pre .envvar, .swagger-section pre .shebang, .swagger-section pre .apache .sqbracket, .swagger-section pre .nginx .built_in, .swagger-section pre .tex .formula, .swagger-section pre .erlang_repl .reserved, .swagger-section pre .prompt, .swagger-section pre .markdown .link_label, .swagger-section pre .vhdl .attribute, .swagger-section pre .clojure .attribute, .swagger-section pre .coffeescript .property { color: #88F; } .swagger-section pre .keyword, .swagger-section pre .id, .swagger-section pre .phpdoc, .swagger-section pre .title, .swagger-section pre .built_in, .swagger-section pre .aggregate, .swagger-section pre .css .tag, .swagger-section pre .javadoctag, .swagger-section pre .phpdoc, .swagger-section pre .yardoctag, .swagger-section pre .smalltalk .class, .swagger-section pre .winutils, .swagger-section pre .bash .variable, .swagger-section pre .apache .tag, .swagger-section pre .go .typename, .swagger-section pre .tex .command, .swagger-section pre .markdown .strong, .swagger-section pre .request, .swagger-section pre .status { font-weight: bold; } .swagger-section pre .markdown .emphasis { font-style: italic; } .swagger-section pre .nginx .built_in { font-weight: normal; } .swagger-section pre .coffeescript .javascript, .swagger-section pre .javascript .xml, .swagger-section pre .tex .formula, .swagger-section pre .xml .javascript, .swagger-section pre .xml .vbscript, .swagger-section pre .xml .css, .swagger-section pre .xml .cdata { opacity: 0.5; } .swagger-section .hljs { display: block; overflow-x: auto; padding: 0.5em; background: #F0F0F0; } .swagger-section .hljs, .swagger-section .hljs-subst { color: #444; } .swagger-section .hljs-keyword, .swagger-section .hljs-attribute, .swagger-section .hljs-selector-tag, .swagger-section .hljs-meta-keyword, .swagger-section .hljs-doctag, .swagger-section .hljs-name { font-weight: bold; } .swagger-section .hljs-built_in, .swagger-section .hljs-literal, .swagger-section .hljs-bullet, .swagger-section .hljs-code, .swagger-section .hljs-addition { color: #1F811F; } .swagger-section .hljs-regexp, .swagger-section .hljs-symbol, .swagger-section .hljs-variable, .swagger-section .hljs-template-variable, .swagger-section .hljs-link, .swagger-section .hljs-selector-attr, .swagger-section .hljs-selector-pseudo { color: #BC6060; } .swagger-section .hljs-type, .swagger-section .hljs-string, .swagger-section .hljs-number, .swagger-section .hljs-selector-id, .swagger-section .hljs-selector-class, .swagger-section .hljs-quote, .swagger-section .hljs-template-tag, .swagger-section .hljs-deletion { color: #880000; } .swagger-section .hljs-title, .swagger-section .hljs-section { color: #880000; font-weight: bold; } .swagger-section .hljs-comment { color: #888888; } .swagger-section .hljs-meta { color: #2B6EA1; } .swagger-section .hljs-emphasis { font-style: italic; } .swagger-section .hljs-strong { font-weight: bold; } .swagger-section .swagger-ui-wrap { line-height: 1; font-family: "Droid Sans", sans-serif; min-width: 760px; max-width: 960px; margin-left: auto; margin-right: auto; /* JSONEditor specific styling */ } .swagger-section .swagger-ui-wrap b, .swagger-section .swagger-ui-wrap strong { font-family: "Droid Sans", sans-serif; font-weight: bold; } .swagger-section .swagger-ui-wrap q, .swagger-section .swagger-ui-wrap blockquote { quotes: none; } .swagger-section .swagger-ui-wrap p { line-height: 1.4em; padding: 0 0 10px; color: #333333; } .swagger-section .swagger-ui-wrap q:before, .swagger-section .swagger-ui-wrap q:after, .swagger-section .swagger-ui-wrap blockquote:before, .swagger-section .swagger-ui-wrap blockquote:after { content: none; } .swagger-section .swagger-ui-wrap .heading_with_menu h1, .swagger-section .swagger-ui-wrap .heading_with_menu h2, .swagger-section .swagger-ui-wrap .heading_with_menu h3, .swagger-section .swagger-ui-wrap .heading_with_menu h4, .swagger-section .swagger-ui-wrap .heading_with_menu h5, .swagger-section .swagger-ui-wrap .heading_with_menu h6 { display: block; clear: none; float: left; -moz-box-sizing: border-box; -webkit-box-sizing: border-box; -ms-box-sizing: border-box; box-sizing: border-box; width: 60%; } .swagger-section .swagger-ui-wrap table { border-collapse: collapse; border-spacing: 0; } .swagger-section .swagger-ui-wrap table thead tr th { padding: 5px; font-size: 0.9em; color: #666666; border-bottom: 1px solid #999999; } .swagger-section .swagger-ui-wrap table tbody tr:last-child td { border-bottom: none; } .swagger-section .swagger-ui-wrap table tbody tr.offset { background-color: #f0f0f0; } .swagger-section .swagger-ui-wrap table tbody tr td { padding: 6px; font-size: 0.9em; border-bottom: 1px solid #cccccc; vertical-align: top; line-height: 1.3em; } .swagger-section .swagger-ui-wrap ol { margin: 0px 0 10px; padding: 0 0 0 18px; list-style-type: decimal; } .swagger-section .swagger-ui-wrap ol li { padding: 5px 0px; font-size: 0.9em; color: #333333; } .swagger-section .swagger-ui-wrap ol, .swagger-section .swagger-ui-wrap ul { list-style: none; } .swagger-section .swagger-ui-wrap h1 a, .swagger-section .swagger-ui-wrap h2 a, .swagger-section .swagger-ui-wrap h3 a, .swagger-section .swagger-ui-wrap h4 a, .swagger-section .swagger-ui-wrap h5 a, .swagger-section .swagger-ui-wrap h6 a { text-decoration: none; } .swagger-section .swagger-ui-wrap h1 a:hover, .swagger-section .swagger-ui-wrap h2 a:hover, .swagger-section .swagger-ui-wrap h3 a:hover, .swagger-section .swagger-ui-wrap h4 a:hover, .swagger-section .swagger-ui-wrap h5 a:hover, .swagger-section .swagger-ui-wrap h6 a:hover { text-decoration: underline; } .swagger-section .swagger-ui-wrap h1 span.divider, .swagger-section .swagger-ui-wrap h2 span.divider, .swagger-section .swagger-ui-wrap h3 span.divider, .swagger-section .swagger-ui-wrap h4 span.divider, .swagger-section .swagger-ui-wrap h5 span.divider, .swagger-section .swagger-ui-wrap h6 span.divider { color: #aaaaaa; } .swagger-section .swagger-ui-wrap a { color: #547f00; } .swagger-section .swagger-ui-wrap a img { border: none; } .swagger-section .swagger-ui-wrap article, .swagger-section .swagger-ui-wrap aside, .swagger-section .swagger-ui-wrap details, .swagger-section .swagger-ui-wrap figcaption, .swagger-section .swagger-ui-wrap figure, .swagger-section .swagger-ui-wrap footer, .swagger-section .swagger-ui-wrap header, .swagger-section .swagger-ui-wrap hgroup, .swagger-section .swagger-ui-wrap menu, .swagger-section .swagger-ui-wrap nav, .swagger-section .swagger-ui-wrap section, .swagger-section .swagger-ui-wrap summary { display: block; } .swagger-section .swagger-ui-wrap pre { font-family: "Anonymous Pro", "Menlo", "Consolas", "Bitstream Vera Sans Mono", "Courier New", monospace; background-color: #fcf6db; border: 1px solid #e5e0c6; padding: 10px; } .swagger-section .swagger-ui-wrap pre code { line-height: 1.6em; background: none; } .swagger-section .swagger-ui-wrap .content > .content-type > div > label { clear: both; display: block; color: #0F6AB4; font-size: 1.1em; margin: 0; padding: 15px 0 5px; } .swagger-section .swagger-ui-wrap .content pre { font-size: 12px; margin-top: 5px; padding: 5px; } .swagger-section .swagger-ui-wrap .icon-btn { cursor: pointer; } .swagger-section .swagger-ui-wrap .info_title { padding-bottom: 10px; font-weight: bold; font-size: 25px; } .swagger-section .swagger-ui-wrap .footer { margin-top: 20px; } .swagger-section .swagger-ui-wrap p.big, .swagger-section .swagger-ui-wrap div.big p { font-size: 1em; margin-bottom: 10px; } .swagger-section .swagger-ui-wrap form.fullwidth ol li.string input, .swagger-section .swagger-ui-wrap form.fullwidth ol li.url input, .swagger-section .swagger-ui-wrap form.fullwidth ol li.text textarea, .swagger-section .swagger-ui-wrap form.fullwidth ol li.numeric input { width: 500px !important; } .swagger-section .swagger-ui-wrap .info_license { padding-bottom: 5px; } .swagger-section .swagger-ui-wrap .info_tos { padding-bottom: 5px; } .swagger-section .swagger-ui-wrap .message-fail { color: #cc0000; } .swagger-section .swagger-ui-wrap .info_url { padding-bottom: 5px; } .swagger-section .swagger-ui-wrap .info_email { padding-bottom: 5px; } .swagger-section .swagger-ui-wrap .info_name { padding-bottom: 5px; } .swagger-section .swagger-ui-wrap .info_description { padding-bottom: 10px; font-size: 15px; } .swagger-section .swagger-ui-wrap .markdown ol li, .swagger-section .swagger-ui-wrap .markdown ul li { padding: 3px 0px; line-height: 1.4em; color: #333333; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.string input, .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.url input, .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.numeric input { display: block; padding: 4px; width: auto; clear: both; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.string input.title, .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.url input.title, .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.numeric input.title { font-size: 1.3em; } .swagger-section .swagger-ui-wrap table.fullwidth { width: 100%; } .swagger-section .swagger-ui-wrap .model-signature { font-family: "Droid Sans", sans-serif; font-size: 1em; line-height: 1.5em; } .swagger-section .swagger-ui-wrap .model-signature .signature-nav a { text-decoration: none; color: #AAA; } .swagger-section .swagger-ui-wrap .model-signature .signature-nav a:hover { text-decoration: underline; color: black; } .swagger-section .swagger-ui-wrap .model-signature .signature-nav .selected { color: black; text-decoration: none; } .swagger-section .swagger-ui-wrap .model-signature .propType { color: #5555aa; } .swagger-section .swagger-ui-wrap .model-signature pre:hover { background-color: #ffffdd; } .swagger-section .swagger-ui-wrap .model-signature pre { font-size: .85em; line-height: 1.2em; overflow: auto; max-height: 200px; cursor: pointer; } .swagger-section .swagger-ui-wrap .model-signature ul.signature-nav { display: block; min-width: 230px; margin: 0; padding: 0; } .swagger-section .swagger-ui-wrap .model-signature ul.signature-nav li:last-child { padding-right: 0; border-right: none; } .swagger-section .swagger-ui-wrap .model-signature ul.signature-nav li { float: left; margin: 0 5px 5px 0; padding: 2px 5px 2px 0; border-right: 1px solid #ddd; } .swagger-section .swagger-ui-wrap .model-signature .propOpt { color: #555; } .swagger-section .swagger-ui-wrap .model-signature .snippet small { font-size: 0.75em; } .swagger-section .swagger-ui-wrap .model-signature .propOptKey { font-style: italic; } .swagger-section .swagger-ui-wrap .model-signature .description .strong { font-weight: bold; color: #000; font-size: .9em; } .swagger-section .swagger-ui-wrap .model-signature .description div { font-size: 0.9em; line-height: 1.5em; margin-left: 1em; } .swagger-section .swagger-ui-wrap .model-signature .description .stronger { font-weight: bold; color: #000; } .swagger-section .swagger-ui-wrap .model-signature .description .propWrap .optionsWrapper { border-spacing: 0; position: absolute; background-color: #ffffff; border: 1px solid #bbbbbb; display: none; font-size: 11px; max-width: 400px; line-height: 30px; color: black; padding: 5px; margin-left: 10px; } .swagger-section .swagger-ui-wrap .model-signature .description .propWrap .optionsWrapper th { text-align: center; background-color: #eeeeee; border: 1px solid #bbbbbb; font-size: 11px; color: #666666; font-weight: bold; padding: 5px; line-height: 15px; } .swagger-section .swagger-ui-wrap .model-signature .description .propWrap .optionsWrapper .optionName { font-weight: bold; } .swagger-section .swagger-ui-wrap .model-signature .description .propDesc.markdown > p:first-child, .swagger-section .swagger-ui-wrap .model-signature .description .propDesc.markdown > p:last-child { display: inline; } .swagger-section .swagger-ui-wrap .model-signature .description .propDesc.markdown > p:not(:first-child):before { display: block; content: ''; } .swagger-section .swagger-ui-wrap .model-signature .description span:last-of-type.propDesc.markdown > p:only-child { margin-right: -3px; } .swagger-section .swagger-ui-wrap .model-signature .propName { font-weight: bold; } .swagger-section .swagger-ui-wrap .model-signature .signature-container { clear: both; } .swagger-section .swagger-ui-wrap .body-textarea { width: 300px; height: 100px; border: 1px solid #aaa; } .swagger-section .swagger-ui-wrap .markdown p code, .swagger-section .swagger-ui-wrap .markdown li code { font-family: "Anonymous Pro", "Menlo", "Consolas", "Bitstream Vera Sans Mono", "Courier New", monospace; background-color: #f0f0f0; color: black; padding: 1px 3px; } .swagger-section .swagger-ui-wrap .required { font-weight: bold; } .swagger-section .swagger-ui-wrap .editor_holder { font-family: "Anonymous Pro", "Menlo", "Consolas", "Bitstream Vera Sans Mono", "Courier New", monospace; font-size: 0.9em; } .swagger-section .swagger-ui-wrap .editor_holder label { font-weight: normal!important; /* JSONEditor uses bold by default for all labels, we revert that back to normal to not give the impression that by default fields are required */ } .swagger-section .swagger-ui-wrap .editor_holder label.required { font-weight: bold!important; } .swagger-section .swagger-ui-wrap input.parameter { width: 300px; border: 1px solid #aaa; } .swagger-section .swagger-ui-wrap h1 { color: black; font-size: 1.5em; line-height: 1.3em; padding: 10px 0 10px 0; font-family: "Droid Sans", sans-serif; font-weight: bold; } .swagger-section .swagger-ui-wrap .heading_with_menu { float: none; clear: both; overflow: hidden; display: block; } .swagger-section .swagger-ui-wrap .heading_with_menu ul { display: block; clear: none; float: right; -moz-box-sizing: border-box; -webkit-box-sizing: border-box; -ms-box-sizing: border-box; box-sizing: border-box; margin-top: 10px; } .swagger-section .swagger-ui-wrap h2 { color: black; font-size: 1.3em; padding: 10px 0 10px 0; } .swagger-section .swagger-ui-wrap h2 a { color: black; } .swagger-section .swagger-ui-wrap h2 span.sub { font-size: 0.7em; color: #999999; font-style: italic; } .swagger-section .swagger-ui-wrap h2 span.sub a { color: #777777; } .swagger-section .swagger-ui-wrap span.weak { color: #666666; } .swagger-section .swagger-ui-wrap .message-success { color: #89BF04; } .swagger-section .swagger-ui-wrap caption, .swagger-section .swagger-ui-wrap th, .swagger-section .swagger-ui-wrap td { text-align: left; font-weight: normal; vertical-align: middle; } .swagger-section .swagger-ui-wrap .code { font-family: "Anonymous Pro", "Menlo", "Consolas", "Bitstream Vera Sans Mono", "Courier New", monospace; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.text textarea { font-family: "Droid Sans", sans-serif; height: 250px; padding: 4px; display: block; clear: both; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.select select { display: block; clear: both; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.boolean { float: none; clear: both; overflow: hidden; display: block; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.boolean label { display: block; float: left; clear: none; margin: 0; padding: 0; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.boolean input { display: block; float: left; clear: none; margin: 0 5px 0 0; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.required label { color: black; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li label { display: block; clear: both; width: auto; padding: 0 0 3px; color: #666666; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li label abbr { padding-left: 3px; color: #888888; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li p.inline-hints { margin-left: 0; font-style: italic; font-size: 0.9em; margin: 0; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.buttons { margin: 0; padding: 0; } .swagger-section .swagger-ui-wrap span.blank, .swagger-section .swagger-ui-wrap span.empty { color: #888888; font-style: italic; } .swagger-section .swagger-ui-wrap .markdown h3 { color: #547f00; } .swagger-section .swagger-ui-wrap .markdown h4 { color: #666666; } .swagger-section .swagger-ui-wrap .markdown pre { font-family: "Anonymous Pro", "Menlo", "Consolas", "Bitstream Vera Sans Mono", "Courier New", monospace; background-color: #fcf6db; border: 1px solid #e5e0c6; padding: 10px; margin: 0 0 10px 0; } .swagger-section .swagger-ui-wrap .markdown pre code { line-height: 1.6em; overflow: auto; } .swagger-section .swagger-ui-wrap div.gist { margin: 20px 0 25px 0 !important; } .swagger-section .swagger-ui-wrap ul#resources { font-family: "Droid Sans", sans-serif; font-size: 0.9em; } .swagger-section .swagger-ui-wrap ul#resources li.resource { border-bottom: 1px solid #dddddd; } .swagger-section .swagger-ui-wrap ul#resources li.resource:hover div.heading h2 a, .swagger-section .swagger-ui-wrap ul#resources li.resource.active div.heading h2 a { color: black; } .swagger-section .swagger-ui-wrap ul#resources li.resource:hover div.heading ul.options li a, .swagger-section .swagger-ui-wrap ul#resources li.resource.active div.heading ul.options li a { color: #555555; } .swagger-section .swagger-ui-wrap ul#resources li.resource:last-child { border-bottom: none; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading { border: 1px solid transparent; float: none; clear: both; overflow: hidden; display: block; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options { overflow: hidden; padding: 0; display: block; clear: none; float: right; margin: 14px 10px 0 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li { float: left; clear: none; margin: 0; padding: 2px 10px; border-right: 1px solid #dddddd; color: #666666; font-size: 0.9em; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li a { color: #aaaaaa; text-decoration: none; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li a:hover { text-decoration: underline; color: black; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li a:hover, .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li a:active, .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li a.active { text-decoration: underline; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li:first-child, .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li.first { padding-left: 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li:last-child, .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li.last { padding-right: 0; border-right: none; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options:first-child, .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options.first { padding-left: 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading h2 { color: #999999; padding-left: 0; display: block; clear: none; float: left; font-family: "Droid Sans", sans-serif; font-weight: bold; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading h2 a { color: #999999; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading h2 a:hover { color: black; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation { float: none; clear: both; overflow: hidden; display: block; margin: 0 0 10px; padding: 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading { float: none; clear: both; overflow: hidden; display: block; margin: 0; padding: 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading h3 { display: block; clear: none; float: left; width: auto; margin: 0; padding: 0; line-height: 1.1em; color: black; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading h3 span.path { padding-left: 10px; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading h3 span.path a { color: black; text-decoration: none; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading h3 span.path a.toggleOperation.deprecated { text-decoration: line-through; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading h3 span.path a:hover { text-decoration: underline; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading h3 span.http_method a { text-transform: uppercase; text-decoration: none; color: white; display: inline-block; width: 50px; font-size: 0.7em; text-align: center; padding: 7px 0 4px; -moz-border-radius: 2px; -webkit-border-radius: 2px; -o-border-radius: 2px; -ms-border-radius: 2px; -khtml-border-radius: 2px; border-radius: 2px; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading h3 span { margin: 0; padding: 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading ul.options { overflow: hidden; padding: 0; display: block; clear: none; float: right; margin: 6px 10px 0 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading ul.options li { float: left; clear: none; margin: 0; padding: 2px 10px; font-size: 0.9em; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading ul.options li a { text-decoration: none; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading ul.options li a .markdown p { color: inherit; padding: 0; line-height: inherit; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading ul.options li.access { color: black; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content { border-top: none; padding: 10px; -moz-border-radius-bottomleft: 6px; -webkit-border-bottom-left-radius: 6px; -o-border-bottom-left-radius: 6px; -ms-border-bottom-left-radius: 6px; -khtml-border-bottom-left-radius: 6px; border-bottom-left-radius: 6px; -moz-border-radius-bottomright: 6px; -webkit-border-bottom-right-radius: 6px; -o-border-bottom-right-radius: 6px; -ms-border-bottom-right-radius: 6px; -khtml-border-bottom-right-radius: 6px; border-bottom-right-radius: 6px; margin: 0 0 20px; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content h4 { font-size: 1.1em; margin: 0; padding: 15px 0 5px; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content div.sandbox_header { float: none; clear: both; overflow: hidden; display: block; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content div.sandbox_header a { padding: 4px 0 0 10px; display: inline-block; font-size: 0.9em; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content div.sandbox_header input.submit { display: block; clear: none; float: left; padding: 6px 8px; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content div.sandbox_header span.response_throbber { background-image: url('../images/throbber.gif'); width: 128px; height: 16px; display: block; clear: none; float: right; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content form input[type='text'].error { outline: 2px solid black; outline-color: #cc0000; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content form select[name='parameterContentType'] { max-width: 300px; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content div.response div.block pre { font-family: "Anonymous Pro", "Menlo", "Consolas", "Bitstream Vera Sans Mono", "Courier New", monospace; padding: 10px; font-size: 0.9em; max-height: 400px; overflow-y: auto; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.heading { background-color: #f9f2e9; border: 1px solid #f0e0ca; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.heading h3 span.http_method a { background-color: #c5862b; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.heading ul.options li { border-right: 1px solid #dddddd; border-right-color: #f0e0ca; color: #c5862b; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.heading ul.options li a { color: #c5862b; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.content { background-color: #faf5ee; border: 1px solid #f0e0ca; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.content h4 { color: #c5862b; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.content div.sandbox_header a { color: #dcb67f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.heading { background-color: #fcffcd; border: 1px solid black; border-color: #ffd20f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.heading h3 span.http_method a { text-transform: uppercase; background-color: #ffd20f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.heading ul.options li { border-right: 1px solid #dddddd; border-right-color: #ffd20f; color: #ffd20f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.heading ul.options li a { color: #ffd20f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.content { background-color: #fcffcd; border: 1px solid black; border-color: #ffd20f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.content h4 { color: #ffd20f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.content div.sandbox_header a { color: #6fc992; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.heading { background-color: #f5e8e8; border: 1px solid #e8c6c7; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.heading h3 span.http_method a { text-transform: uppercase; background-color: #a41e22; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.heading ul.options li { border-right: 1px solid #dddddd; border-right-color: #e8c6c7; color: #a41e22; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.heading ul.options li a { color: #a41e22; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.content { background-color: #f7eded; border: 1px solid #e8c6c7; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.content h4 { color: #a41e22; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.content div.sandbox_header a { color: #c8787a; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.heading { background-color: #e7f6ec; border: 1px solid #c3e8d1; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.heading h3 span.http_method a { background-color: #10a54a; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.heading ul.options li { border-right: 1px solid #dddddd; border-right-color: #c3e8d1; color: #10a54a; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.heading ul.options li a { color: #10a54a; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.content { background-color: #ebf7f0; border: 1px solid #c3e8d1; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.content h4 { color: #10a54a; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.content div.sandbox_header a { color: #6fc992; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.heading { background-color: #FCE9E3; border: 1px solid #F5D5C3; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.heading h3 span.http_method a { background-color: #D38042; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.heading ul.options li { border-right: 1px solid #dddddd; border-right-color: #f0cecb; color: #D38042; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.heading ul.options li a { color: #D38042; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.content { background-color: #faf0ef; border: 1px solid #f0cecb; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.content h4 { color: #D38042; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.content div.sandbox_header a { color: #dcb67f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.heading { background-color: #e7f0f7; border: 1px solid #c3d9ec; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.heading h3 span.http_method a { background-color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.heading ul.options li { border-right: 1px solid #dddddd; border-right-color: #c3d9ec; color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.heading ul.options li a { color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.content { background-color: #ebf3f9; border: 1px solid #c3d9ec; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.content h4 { color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.content div.sandbox_header a { color: #6fa5d2; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.options div.heading { background-color: #e7f0f7; border: 1px solid #c3d9ec; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.options div.heading h3 span.http_method a { background-color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.options div.heading ul.options li { border-right: 1px solid #dddddd; border-right-color: #c3d9ec; color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.options div.heading ul.options li a { color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.options div.content { background-color: #ebf3f9; border: 1px solid #c3d9ec; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.options div.content h4 { color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.options div.content div.sandbox_header a { color: #6fa5d2; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.content, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.content, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.content, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.content, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.content, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.content { border-top: none; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.heading ul.options li:last-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.heading ul.options li:last-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.heading ul.options li:last-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.heading ul.options li:last-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.heading ul.options li:last-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.heading ul.options li:last-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.heading ul.options li.last, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.heading ul.options li.last, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.heading ul.options li.last, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.heading ul.options li.last, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.heading ul.options li.last, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.heading ul.options li.last { padding-right: 0; border-right: none; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations ul.options li a:hover, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations ul.options li a:active, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations ul.options li a.active { text-decoration: underline; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations ul.options li:first-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations ul.options li.first { padding-left: 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations:first-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations.first { padding-left: 0; } .swagger-section .swagger-ui-wrap p#colophon { margin: 0 15px 40px 15px; padding: 10px 0; font-size: 0.8em; border-top: 1px solid #dddddd; font-family: "Droid Sans", sans-serif; color: #999999; font-style: italic; } .swagger-section .swagger-ui-wrap p#colophon a { text-decoration: none; color: #547f00; } .swagger-section .swagger-ui-wrap h3 { color: black; font-size: 1.1em; padding: 10px 0 10px 0; } .swagger-section .swagger-ui-wrap .markdown ol, .swagger-section .swagger-ui-wrap .markdown ul { font-family: "Droid Sans", sans-serif; margin: 5px 0 10px; padding: 0 0 0 18px; list-style-type: disc; } .swagger-section .swagger-ui-wrap form.form_box { background-color: #ebf3f9; border: 1px solid #c3d9ec; padding: 10px; } .swagger-section .swagger-ui-wrap form.form_box label { color: #0f6ab4 !important; } .swagger-section .swagger-ui-wrap form.form_box input[type=submit] { display: block; padding: 10px; } .swagger-section .swagger-ui-wrap form.form_box p.weak { font-size: 0.8em; } .swagger-section .swagger-ui-wrap form.form_box p { font-size: 0.9em; padding: 0 0 15px; color: #7e7b6d; } .swagger-section .swagger-ui-wrap form.form_box p a { color: #646257; } .swagger-section .swagger-ui-wrap form.form_box p strong { color: black; } .swagger-section .swagger-ui-wrap .operation-status td.markdown > p:last-child { padding-bottom: 0; } .swagger-section .title { font-style: bold; } .swagger-section .secondary_form { display: none; } .swagger-section .main_image { display: block; margin-left: auto; margin-right: auto; } .swagger-section .oauth_body { margin-left: 100px; margin-right: 100px; } .swagger-section .oauth_submit { text-align: center; display: inline-block; } .swagger-section .authorize-wrapper { margin: 15px 0 10px; } .swagger-section .authorize-wrapper_operation { float: right; } .swagger-section .authorize__btn:hover { text-decoration: underline; cursor: pointer; } .swagger-section .authorize__btn_operation:hover .authorize-scopes { display: block; } .swagger-section .authorize-scopes { position: absolute; margin-top: 20px; background: #FFF; border: 1px solid #ccc; border-radius: 5px; display: none; font-size: 13px; max-width: 300px; line-height: 30px; color: black; padding: 5px; } .swagger-section .authorize-scopes .authorize__scope { text-decoration: none; } .swagger-section .authorize__btn_operation { height: 18px; vertical-align: middle; display: inline-block; background: url(../images/explorer_icons.png) no-repeat; } .swagger-section .authorize__btn_operation_login { background-position: 0 0; width: 18px; margin-top: -6px; margin-left: 4px; } .swagger-section .authorize__btn_operation_logout { background-position: -30px 0; width: 18px; margin-top: -6px; margin-left: 4px; } .swagger-section #auth_container { color: #fff; display: inline-block; border: none; padding: 5px; width: 87px; height: 13px; } .swagger-section #auth_container .authorize__btn { color: #fff; } .swagger-section .auth_container { padding: 0 0 10px; margin-bottom: 5px; border-bottom: solid 1px #CCC; font-size: 0.9em; } .swagger-section .auth_container .auth__title { color: #547f00; font-size: 1.2em; } .swagger-section .auth_container .basic_auth__label { display: inline-block; width: 60px; } .swagger-section .auth_container .auth__description { color: #999999; margin-bottom: 5px; } .swagger-section .auth_container .auth__button { margin-top: 10px; height: 30px; } .swagger-section .auth_container .key_auth__field { margin: 5px 0; } .swagger-section .auth_container .key_auth__label { display: inline-block; width: 60px; } .swagger-section .api-popup-dialog { position: absolute; display: none; } .swagger-section .api-popup-dialog-wrapper { z-index: 1000; width: 500px; background: #FFF; padding: 20px; border: 1px solid #ccc; border-radius: 5px; font-size: 13px; color: #777; position: fixed; top: 50%; left: 50%; transform: translate(-50%, -50%); } .swagger-section .api-popup-dialog-shadow { position: fixed; top: 0; left: 0; width: 100%; height: 100%; opacity: 0.2; background-color: gray; z-index: 900; } .swagger-section .api-popup-dialog .api-popup-title { font-size: 24px; padding: 10px 0; } .swagger-section .api-popup-dialog .api-popup-title { font-size: 24px; padding: 10px 0; } .swagger-section .api-popup-dialog .error-msg { padding-left: 5px; padding-bottom: 5px; } .swagger-section .api-popup-dialog .api-popup-content { max-height: 500px; overflow-y: auto; } .swagger-section .api-popup-dialog .api-popup-authbtn { height: 30px; } .swagger-section .api-popup-dialog .api-popup-cancel { height: 30px; } .swagger-section .api-popup-scopes { padding: 10px 20px; } .swagger-section .api-popup-scopes li { padding: 5px 0; line-height: 20px; } .swagger-section .api-popup-scopes li input { position: relative; top: 2px; } .swagger-section .api-popup-scopes .api-scope-desc { padding-left: 20px; font-style: italic; } .swagger-section .api-popup-actions { padding-top: 10px; } #header { display: none; } .swagger-section .swagger-ui-wrap .model-signature pre { max-height: none; } .swagger-section .swagger-ui-wrap .body-textarea { width: 100px; } .swagger-section .swagger-ui-wrap input.parameter { width: 100px; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options { display: none; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints { display: block !important; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content { display: block !important; } ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/css/reset.css ================================================ /* http://meyerweb.com/eric/tools/css/reset/ v2.0 | 20110126 */ html, body, div, span, applet, object, iframe, h1, h2, h3, h4, h5, h6, p, blockquote, pre, a, abbr, acronym, address, big, cite, code, del, dfn, em, img, ins, kbd, q, s, samp, small, strike, strong, sub, sup, tt, var, b, u, i, center, dl, dt, dd, ol, ul, li, fieldset, form, label, legend, table, caption, tbody, tfoot, thead, tr, th, td, article, aside, canvas, details, embed, figure, figcaption, footer, header, hgroup, menu, nav, output, ruby, section, summary, time, mark, audio, video { margin: 0; padding: 0; border: 0; font-size: 100%; font: inherit; vertical-align: baseline; } /* HTML5 display-role reset for older browsers */ article, aside, details, figcaption, figure, footer, header, hgroup, menu, nav, section { display: block; } body { line-height: 1; } ol, ul { list-style: none; } blockquote, q { quotes: none; } blockquote:before, blockquote:after, q:before, q:after { content: ''; content: none; } table { border-collapse: collapse; border-spacing: 0; } ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/css/screen.css ================================================ /* Original style from softwaremaniacs.org (c) Ivan Sagalaev */ .swagger-section pre code { display: block; padding: 0.5em; background: #F0F0F0; } .swagger-section pre code, .swagger-section pre .subst, .swagger-section pre .tag .title, .swagger-section pre .lisp .title, .swagger-section pre .clojure .built_in, .swagger-section pre .nginx .title { color: black; } .swagger-section pre .string, .swagger-section pre .title, .swagger-section pre .constant, .swagger-section pre .parent, .swagger-section pre .tag .value, .swagger-section pre .rules .value, .swagger-section pre .rules .value .number, .swagger-section pre .preprocessor, .swagger-section pre .ruby .symbol, .swagger-section pre .ruby .symbol .string, .swagger-section pre .aggregate, .swagger-section pre .template_tag, .swagger-section pre .django .variable, .swagger-section pre .smalltalk .class, .swagger-section pre .addition, .swagger-section pre .flow, .swagger-section pre .stream, .swagger-section pre .bash .variable, .swagger-section pre .apache .tag, .swagger-section pre .apache .cbracket, .swagger-section pre .tex .command, .swagger-section pre .tex .special, .swagger-section pre .erlang_repl .function_or_atom, .swagger-section pre .markdown .header { color: #800; } .swagger-section pre .comment, .swagger-section pre .annotation, .swagger-section pre .template_comment, .swagger-section pre .diff .header, .swagger-section pre .chunk, .swagger-section pre .markdown .blockquote { color: #888; } .swagger-section pre .number, .swagger-section pre .date, .swagger-section pre .regexp, .swagger-section pre .literal, .swagger-section pre .smalltalk .symbol, .swagger-section pre .smalltalk .char, .swagger-section pre .go .constant, .swagger-section pre .change, .swagger-section pre .markdown .bullet, .swagger-section pre .markdown .link_url { color: #080; } .swagger-section pre .label, .swagger-section pre .javadoc, .swagger-section pre .ruby .string, .swagger-section pre .decorator, .swagger-section pre .filter .argument, .swagger-section pre .localvars, .swagger-section pre .array, .swagger-section pre .attr_selector, .swagger-section pre .important, .swagger-section pre .pseudo, .swagger-section pre .pi, .swagger-section pre .doctype, .swagger-section pre .deletion, .swagger-section pre .envvar, .swagger-section pre .shebang, .swagger-section pre .apache .sqbracket, .swagger-section pre .nginx .built_in, .swagger-section pre .tex .formula, .swagger-section pre .erlang_repl .reserved, .swagger-section pre .prompt, .swagger-section pre .markdown .link_label, .swagger-section pre .vhdl .attribute, .swagger-section pre .clojure .attribute, .swagger-section pre .coffeescript .property { color: #88F; } .swagger-section pre .keyword, .swagger-section pre .id, .swagger-section pre .phpdoc, .swagger-section pre .title, .swagger-section pre .built_in, .swagger-section pre .aggregate, .swagger-section pre .css .tag, .swagger-section pre .javadoctag, .swagger-section pre .phpdoc, .swagger-section pre .yardoctag, .swagger-section pre .smalltalk .class, .swagger-section pre .winutils, .swagger-section pre .bash .variable, .swagger-section pre .apache .tag, .swagger-section pre .go .typename, .swagger-section pre .tex .command, .swagger-section pre .markdown .strong, .swagger-section pre .request, .swagger-section pre .status { font-weight: bold; } .swagger-section pre .markdown .emphasis { font-style: italic; } .swagger-section pre .nginx .built_in { font-weight: normal; } .swagger-section pre .coffeescript .javascript, .swagger-section pre .javascript .xml, .swagger-section pre .tex .formula, .swagger-section pre .xml .javascript, .swagger-section pre .xml .vbscript, .swagger-section pre .xml .css, .swagger-section pre .xml .cdata { opacity: 0.5; } .swagger-section .hljs { display: block; overflow-x: auto; padding: 0.5em; background: #F0F0F0; } .swagger-section .hljs, .swagger-section .hljs-subst { color: #444; } .swagger-section .hljs-keyword, .swagger-section .hljs-attribute, .swagger-section .hljs-selector-tag, .swagger-section .hljs-meta-keyword, .swagger-section .hljs-doctag, .swagger-section .hljs-name { font-weight: bold; } .swagger-section .hljs-built_in, .swagger-section .hljs-literal, .swagger-section .hljs-bullet, .swagger-section .hljs-code, .swagger-section .hljs-addition { color: #1F811F; } .swagger-section .hljs-regexp, .swagger-section .hljs-symbol, .swagger-section .hljs-variable, .swagger-section .hljs-template-variable, .swagger-section .hljs-link, .swagger-section .hljs-selector-attr, .swagger-section .hljs-selector-pseudo { color: #BC6060; } .swagger-section .hljs-type, .swagger-section .hljs-string, .swagger-section .hljs-number, .swagger-section .hljs-selector-id, .swagger-section .hljs-selector-class, .swagger-section .hljs-quote, .swagger-section .hljs-template-tag, .swagger-section .hljs-deletion { color: #880000; } .swagger-section .hljs-title, .swagger-section .hljs-section { color: #880000; font-weight: bold; } .swagger-section .hljs-comment { color: #888888; } .swagger-section .hljs-meta { color: #2B6EA1; } .swagger-section .hljs-emphasis { font-style: italic; } .swagger-section .hljs-strong { font-weight: bold; } .swagger-section .swagger-ui-wrap { line-height: 1; font-family: "Droid Sans", sans-serif; min-width: 760px; max-width: 960px; margin-left: auto; margin-right: auto; /* JSONEditor specific styling */ } .swagger-section .swagger-ui-wrap b, .swagger-section .swagger-ui-wrap strong { font-family: "Droid Sans", sans-serif; font-weight: bold; } .swagger-section .swagger-ui-wrap q, .swagger-section .swagger-ui-wrap blockquote { quotes: none; } .swagger-section .swagger-ui-wrap p { line-height: 1.4em; padding: 0 0 10px; color: #333333; } .swagger-section .swagger-ui-wrap q:before, .swagger-section .swagger-ui-wrap q:after, .swagger-section .swagger-ui-wrap blockquote:before, .swagger-section .swagger-ui-wrap blockquote:after { content: none; } .swagger-section .swagger-ui-wrap .heading_with_menu h1, .swagger-section .swagger-ui-wrap .heading_with_menu h2, .swagger-section .swagger-ui-wrap .heading_with_menu h3, .swagger-section .swagger-ui-wrap .heading_with_menu h4, .swagger-section .swagger-ui-wrap .heading_with_menu h5, .swagger-section .swagger-ui-wrap .heading_with_menu h6 { display: block; clear: none; float: left; -moz-box-sizing: border-box; -webkit-box-sizing: border-box; -ms-box-sizing: border-box; box-sizing: border-box; width: 60%; } .swagger-section .swagger-ui-wrap table { border-collapse: collapse; border-spacing: 0; } .swagger-section .swagger-ui-wrap table thead tr th { padding: 5px; font-size: 0.9em; color: #666666; border-bottom: 1px solid #999999; } .swagger-section .swagger-ui-wrap table tbody tr:last-child td { border-bottom: none; } .swagger-section .swagger-ui-wrap table tbody tr.offset { background-color: #f0f0f0; } .swagger-section .swagger-ui-wrap table tbody tr td { padding: 6px; font-size: 0.9em; border-bottom: 1px solid #cccccc; vertical-align: top; line-height: 1.3em; } .swagger-section .swagger-ui-wrap ol { margin: 0px 0 10px; padding: 0 0 0 18px; list-style-type: decimal; } .swagger-section .swagger-ui-wrap ol li { padding: 5px 0px; font-size: 0.9em; color: #333333; } .swagger-section .swagger-ui-wrap ol, .swagger-section .swagger-ui-wrap ul { list-style: none; } .swagger-section .swagger-ui-wrap h1 a, .swagger-section .swagger-ui-wrap h2 a, .swagger-section .swagger-ui-wrap h3 a, .swagger-section .swagger-ui-wrap h4 a, .swagger-section .swagger-ui-wrap h5 a, .swagger-section .swagger-ui-wrap h6 a { text-decoration: none; } .swagger-section .swagger-ui-wrap h1 a:hover, .swagger-section .swagger-ui-wrap h2 a:hover, .swagger-section .swagger-ui-wrap h3 a:hover, .swagger-section .swagger-ui-wrap h4 a:hover, .swagger-section .swagger-ui-wrap h5 a:hover, .swagger-section .swagger-ui-wrap h6 a:hover { text-decoration: underline; } .swagger-section .swagger-ui-wrap h1 span.divider, .swagger-section .swagger-ui-wrap h2 span.divider, .swagger-section .swagger-ui-wrap h3 span.divider, .swagger-section .swagger-ui-wrap h4 span.divider, .swagger-section .swagger-ui-wrap h5 span.divider, .swagger-section .swagger-ui-wrap h6 span.divider { color: #aaaaaa; } .swagger-section .swagger-ui-wrap a { color: #547f00; } .swagger-section .swagger-ui-wrap a img { border: none; } .swagger-section .swagger-ui-wrap article, .swagger-section .swagger-ui-wrap aside, .swagger-section .swagger-ui-wrap details, .swagger-section .swagger-ui-wrap figcaption, .swagger-section .swagger-ui-wrap figure, .swagger-section .swagger-ui-wrap footer, .swagger-section .swagger-ui-wrap header, .swagger-section .swagger-ui-wrap hgroup, .swagger-section .swagger-ui-wrap menu, .swagger-section .swagger-ui-wrap nav, .swagger-section .swagger-ui-wrap section, .swagger-section .swagger-ui-wrap summary { display: block; } .swagger-section .swagger-ui-wrap pre { font-family: "Anonymous Pro", "Menlo", "Consolas", "Bitstream Vera Sans Mono", "Courier New", monospace; background-color: #fcf6db; border: 1px solid #e5e0c6; padding: 10px; } .swagger-section .swagger-ui-wrap pre code { line-height: 1.6em; background: none; } .swagger-section .swagger-ui-wrap .content > .content-type > div > label { clear: both; display: block; color: #0F6AB4; font-size: 1.1em; margin: 0; padding: 15px 0 5px; } .swagger-section .swagger-ui-wrap .content pre { font-size: 12px; margin-top: 5px; padding: 5px; } .swagger-section .swagger-ui-wrap .icon-btn { cursor: pointer; } .swagger-section .swagger-ui-wrap .info_title { padding-bottom: 10px; font-weight: bold; font-size: 25px; } .swagger-section .swagger-ui-wrap .footer { margin-top: 20px; } .swagger-section .swagger-ui-wrap p.big, .swagger-section .swagger-ui-wrap div.big p { font-size: 1em; margin-bottom: 10px; } .swagger-section .swagger-ui-wrap form.fullwidth ol li.string input, .swagger-section .swagger-ui-wrap form.fullwidth ol li.url input, .swagger-section .swagger-ui-wrap form.fullwidth ol li.text textarea, .swagger-section .swagger-ui-wrap form.fullwidth ol li.numeric input { width: 500px !important; } .swagger-section .swagger-ui-wrap .info_license { padding-bottom: 5px; } .swagger-section .swagger-ui-wrap .info_tos { padding-bottom: 5px; } .swagger-section .swagger-ui-wrap .message-fail { color: #cc0000; } .swagger-section .swagger-ui-wrap .info_url { padding-bottom: 5px; } .swagger-section .swagger-ui-wrap .info_email { padding-bottom: 5px; } .swagger-section .swagger-ui-wrap .info_name { padding-bottom: 5px; } .swagger-section .swagger-ui-wrap .info_description { padding-bottom: 10px; font-size: 15px; } .swagger-section .swagger-ui-wrap .markdown ol li, .swagger-section .swagger-ui-wrap .markdown ul li { padding: 3px 0px; line-height: 1.4em; color: #333333; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.string input, .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.url input, .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.numeric input { display: block; padding: 4px; width: auto; clear: both; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.string input.title, .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.url input.title, .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.numeric input.title { font-size: 1.3em; } .swagger-section .swagger-ui-wrap table.fullwidth { width: 100%; } .swagger-section .swagger-ui-wrap .model-signature { font-family: "Droid Sans", sans-serif; font-size: 1em; line-height: 1.5em; } .swagger-section .swagger-ui-wrap .model-signature .signature-nav a { text-decoration: none; color: #AAA; } .swagger-section .swagger-ui-wrap .model-signature .signature-nav a:hover { text-decoration: underline; color: black; } .swagger-section .swagger-ui-wrap .model-signature .signature-nav .selected { color: black; text-decoration: none; } .swagger-section .swagger-ui-wrap .model-signature .propType { color: #5555aa; } .swagger-section .swagger-ui-wrap .model-signature pre:hover { background-color: #ffffdd; } .swagger-section .swagger-ui-wrap .model-signature pre { font-size: .85em; line-height: 1.2em; overflow: auto; max-height: 200px; cursor: pointer; } .swagger-section .swagger-ui-wrap .model-signature ul.signature-nav { display: block; min-width: 230px; margin: 0; padding: 0; } .swagger-section .swagger-ui-wrap .model-signature ul.signature-nav li:last-child { padding-right: 0; border-right: none; } .swagger-section .swagger-ui-wrap .model-signature ul.signature-nav li { float: left; margin: 0 5px 5px 0; padding: 2px 5px 2px 0; border-right: 1px solid #ddd; } .swagger-section .swagger-ui-wrap .model-signature .propOpt { color: #555; } .swagger-section .swagger-ui-wrap .model-signature .snippet small { font-size: 0.75em; } .swagger-section .swagger-ui-wrap .model-signature .propOptKey { font-style: italic; } .swagger-section .swagger-ui-wrap .model-signature .description .strong { font-weight: bold; color: #000; font-size: .9em; } .swagger-section .swagger-ui-wrap .model-signature .description div { font-size: 0.9em; line-height: 1.5em; margin-left: 1em; } .swagger-section .swagger-ui-wrap .model-signature .description .stronger { font-weight: bold; color: #000; } .swagger-section .swagger-ui-wrap .model-signature .description .propWrap .optionsWrapper { border-spacing: 0; position: absolute; background-color: #ffffff; border: 1px solid #bbbbbb; display: none; font-size: 11px; max-width: 400px; line-height: 30px; color: black; padding: 5px; margin-left: 10px; } .swagger-section .swagger-ui-wrap .model-signature .description .propWrap .optionsWrapper th { text-align: center; background-color: #eeeeee; border: 1px solid #bbbbbb; font-size: 11px; color: #666666; font-weight: bold; padding: 5px; line-height: 15px; } .swagger-section .swagger-ui-wrap .model-signature .description .propWrap .optionsWrapper .optionName { font-weight: bold; } .swagger-section .swagger-ui-wrap .model-signature .description .propDesc.markdown > p:first-child, .swagger-section .swagger-ui-wrap .model-signature .description .propDesc.markdown > p:last-child { display: inline; } .swagger-section .swagger-ui-wrap .model-signature .description .propDesc.markdown > p:not(:first-child):before { display: block; content: ''; } .swagger-section .swagger-ui-wrap .model-signature .description span:last-of-type.propDesc.markdown > p:only-child { margin-right: -3px; } .swagger-section .swagger-ui-wrap .model-signature .propName { font-weight: bold; } .swagger-section .swagger-ui-wrap .model-signature .signature-container { clear: both; } .swagger-section .swagger-ui-wrap .body-textarea { width: 300px; height: 100px; border: 1px solid #aaa; } .swagger-section .swagger-ui-wrap .markdown p code, .swagger-section .swagger-ui-wrap .markdown li code { font-family: "Anonymous Pro", "Menlo", "Consolas", "Bitstream Vera Sans Mono", "Courier New", monospace; background-color: #f0f0f0; color: black; padding: 1px 3px; } .swagger-section .swagger-ui-wrap .required { font-weight: bold; } .swagger-section .swagger-ui-wrap .editor_holder { font-family: "Anonymous Pro", "Menlo", "Consolas", "Bitstream Vera Sans Mono", "Courier New", monospace; font-size: 0.9em; } .swagger-section .swagger-ui-wrap .editor_holder label { font-weight: normal!important; /* JSONEditor uses bold by default for all labels, we revert that back to normal to not give the impression that by default fields are required */ } .swagger-section .swagger-ui-wrap .editor_holder label.required { font-weight: bold!important; } .swagger-section .swagger-ui-wrap input.parameter { width: 300px; border: 1px solid #aaa; } .swagger-section .swagger-ui-wrap h1 { color: black; font-size: 1.5em; line-height: 1.3em; padding: 10px 0 10px 0; font-family: "Droid Sans", sans-serif; font-weight: bold; } .swagger-section .swagger-ui-wrap .heading_with_menu { float: none; clear: both; overflow: hidden; display: block; } .swagger-section .swagger-ui-wrap .heading_with_menu ul { display: block; clear: none; float: right; -moz-box-sizing: border-box; -webkit-box-sizing: border-box; -ms-box-sizing: border-box; box-sizing: border-box; margin-top: 10px; } .swagger-section .swagger-ui-wrap h2 { color: black; font-size: 1.3em; padding: 10px 0 10px 0; } .swagger-section .swagger-ui-wrap h2 a { color: black; } .swagger-section .swagger-ui-wrap h2 span.sub { font-size: 0.7em; color: #999999; font-style: italic; } .swagger-section .swagger-ui-wrap h2 span.sub a { color: #777777; } .swagger-section .swagger-ui-wrap span.weak { color: #666666; } .swagger-section .swagger-ui-wrap .message-success { color: #89BF04; } .swagger-section .swagger-ui-wrap caption, .swagger-section .swagger-ui-wrap th, .swagger-section .swagger-ui-wrap td { text-align: left; font-weight: normal; vertical-align: middle; } .swagger-section .swagger-ui-wrap .code { font-family: "Anonymous Pro", "Menlo", "Consolas", "Bitstream Vera Sans Mono", "Courier New", monospace; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.text textarea { font-family: "Droid Sans", sans-serif; height: 250px; padding: 4px; display: block; clear: both; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.select select { display: block; clear: both; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.boolean { float: none; clear: both; overflow: hidden; display: block; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.boolean label { display: block; float: left; clear: none; margin: 0; padding: 0; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.boolean input { display: block; float: left; clear: none; margin: 0 5px 0 0; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li.required label { color: black; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li label { display: block; clear: both; width: auto; padding: 0 0 3px; color: #666666; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li label abbr { padding-left: 3px; color: #888888; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.inputs ol li p.inline-hints { margin-left: 0; font-style: italic; font-size: 0.9em; margin: 0; } .swagger-section .swagger-ui-wrap form.formtastic fieldset.buttons { margin: 0; padding: 0; } .swagger-section .swagger-ui-wrap span.blank, .swagger-section .swagger-ui-wrap span.empty { color: #888888; font-style: italic; } .swagger-section .swagger-ui-wrap .markdown h3 { color: #547f00; } .swagger-section .swagger-ui-wrap .markdown h4 { color: #666666; } .swagger-section .swagger-ui-wrap .markdown pre { font-family: "Anonymous Pro", "Menlo", "Consolas", "Bitstream Vera Sans Mono", "Courier New", monospace; background-color: #fcf6db; border: 1px solid #e5e0c6; padding: 10px; margin: 0 0 10px 0; } .swagger-section .swagger-ui-wrap .markdown pre code { line-height: 1.6em; overflow: auto; } .swagger-section .swagger-ui-wrap div.gist { margin: 20px 0 25px 0 !important; } .swagger-section .swagger-ui-wrap ul#resources { font-family: "Droid Sans", sans-serif; font-size: 0.9em; } .swagger-section .swagger-ui-wrap ul#resources li.resource { border-bottom: 1px solid #dddddd; } .swagger-section .swagger-ui-wrap ul#resources li.resource:hover div.heading h2 a, .swagger-section .swagger-ui-wrap ul#resources li.resource.active div.heading h2 a { color: black; } .swagger-section .swagger-ui-wrap ul#resources li.resource:hover div.heading ul.options li a, .swagger-section .swagger-ui-wrap ul#resources li.resource.active div.heading ul.options li a { color: #555555; } .swagger-section .swagger-ui-wrap ul#resources li.resource:last-child { border-bottom: none; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading { border: 1px solid transparent; float: none; clear: both; overflow: hidden; display: block; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options { overflow: hidden; padding: 0; display: block; clear: none; float: right; margin: 14px 10px 0 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li { float: left; clear: none; margin: 0; padding: 2px 10px; border-right: 1px solid #dddddd; color: #666666; font-size: 0.9em; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li a { color: #aaaaaa; text-decoration: none; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li a:hover { text-decoration: underline; color: black; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li a:hover, .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li a:active, .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li a.active { text-decoration: underline; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li:first-child, .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li.first { padding-left: 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li:last-child, .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options li.last { padding-right: 0; border-right: none; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options:first-child, .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading ul.options.first { padding-left: 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading h2 { color: #999999; padding-left: 0; display: block; clear: none; float: left; font-family: "Droid Sans", sans-serif; font-weight: bold; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading h2 a { color: #999999; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading h2 a:hover { color: black; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation { float: none; clear: both; overflow: hidden; display: block; margin: 0 0 10px; padding: 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading { float: none; clear: both; overflow: hidden; display: block; margin: 0; padding: 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading h3 { display: block; clear: none; float: left; width: auto; margin: 0; padding: 0; line-height: 1.1em; color: black; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading h3 span.path { padding-left: 10px; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading h3 span.path a { color: black; text-decoration: none; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading h3 span.path a.toggleOperation.deprecated { text-decoration: line-through; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading h3 span.path a:hover { text-decoration: underline; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading h3 span.http_method a { text-transform: uppercase; text-decoration: none; color: white; display: inline-block; width: 50px; font-size: 0.7em; text-align: center; padding: 7px 0 4px; -moz-border-radius: 2px; -webkit-border-radius: 2px; -o-border-radius: 2px; -ms-border-radius: 2px; -khtml-border-radius: 2px; border-radius: 2px; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading h3 span { margin: 0; padding: 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading ul.options { overflow: hidden; padding: 0; display: block; clear: none; float: right; margin: 6px 10px 0 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading ul.options li { float: left; clear: none; margin: 0; padding: 2px 10px; font-size: 0.9em; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading ul.options li a { text-decoration: none; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading ul.options li a .markdown p { color: inherit; padding: 0; line-height: inherit; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.heading ul.options li.access { color: black; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content { border-top: none; padding: 10px; -moz-border-radius-bottomleft: 6px; -webkit-border-bottom-left-radius: 6px; -o-border-bottom-left-radius: 6px; -ms-border-bottom-left-radius: 6px; -khtml-border-bottom-left-radius: 6px; border-bottom-left-radius: 6px; -moz-border-radius-bottomright: 6px; -webkit-border-bottom-right-radius: 6px; -o-border-bottom-right-radius: 6px; -ms-border-bottom-right-radius: 6px; -khtml-border-bottom-right-radius: 6px; border-bottom-right-radius: 6px; margin: 0 0 20px; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content h4 { font-size: 1.1em; margin: 0; padding: 15px 0 5px; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content div.sandbox_header { float: none; clear: both; overflow: hidden; display: block; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content div.sandbox_header a { padding: 4px 0 0 10px; display: inline-block; font-size: 0.9em; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content div.sandbox_header input.submit { display: block; clear: none; float: left; padding: 6px 8px; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content div.sandbox_header span.response_throbber { background-image: url('../images/throbber.gif'); width: 128px; height: 16px; display: block; clear: none; float: right; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content form input[type='text'].error { outline: 2px solid black; outline-color: #cc0000; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content form select[name='parameterContentType'] { max-width: 300px; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation div.content div.response div.block pre { font-family: "Anonymous Pro", "Menlo", "Consolas", "Bitstream Vera Sans Mono", "Courier New", monospace; padding: 10px; font-size: 0.9em; max-height: 400px; overflow-y: auto; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.heading { background-color: #f9f2e9; border: 1px solid #f0e0ca; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.heading h3 span.http_method a { background-color: #c5862b; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.heading ul.options li { border-right: 1px solid #dddddd; border-right-color: #f0e0ca; color: #c5862b; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.heading ul.options li a { color: #c5862b; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.content { background-color: #faf5ee; border: 1px solid #f0e0ca; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.content h4 { color: #c5862b; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.content div.sandbox_header a { color: #dcb67f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.heading { background-color: #fcffcd; border: 1px solid black; border-color: #ffd20f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.heading h3 span.http_method a { text-transform: uppercase; background-color: #ffd20f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.heading ul.options li { border-right: 1px solid #dddddd; border-right-color: #ffd20f; color: #ffd20f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.heading ul.options li a { color: #ffd20f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.content { background-color: #fcffcd; border: 1px solid black; border-color: #ffd20f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.content h4 { color: #ffd20f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.content div.sandbox_header a { color: #6fc992; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.heading { background-color: #f5e8e8; border: 1px solid #e8c6c7; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.heading h3 span.http_method a { text-transform: uppercase; background-color: #a41e22; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.heading ul.options li { border-right: 1px solid #dddddd; border-right-color: #e8c6c7; color: #a41e22; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.heading ul.options li a { color: #a41e22; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.content { background-color: #f7eded; border: 1px solid #e8c6c7; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.content h4 { color: #a41e22; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.content div.sandbox_header a { color: #c8787a; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.heading { background-color: #e7f6ec; border: 1px solid #c3e8d1; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.heading h3 span.http_method a { background-color: #10a54a; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.heading ul.options li { border-right: 1px solid #dddddd; border-right-color: #c3e8d1; color: #10a54a; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.heading ul.options li a { color: #10a54a; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.content { background-color: #ebf7f0; border: 1px solid #c3e8d1; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.content h4 { color: #10a54a; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.content div.sandbox_header a { color: #6fc992; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.heading { background-color: #FCE9E3; border: 1px solid #F5D5C3; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.heading h3 span.http_method a { background-color: #D38042; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.heading ul.options li { border-right: 1px solid #dddddd; border-right-color: #f0cecb; color: #D38042; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.heading ul.options li a { color: #D38042; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.content { background-color: #faf0ef; border: 1px solid #f0cecb; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.content h4 { color: #D38042; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.content div.sandbox_header a { color: #dcb67f; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.heading { background-color: #e7f0f7; border: 1px solid #c3d9ec; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.heading h3 span.http_method a { background-color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.heading ul.options li { border-right: 1px solid #dddddd; border-right-color: #c3d9ec; color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.heading ul.options li a { color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.content { background-color: #ebf3f9; border: 1px solid #c3d9ec; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.content h4 { color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.content div.sandbox_header a { color: #6fa5d2; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.options div.heading { background-color: #e7f0f7; border: 1px solid #c3d9ec; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.options div.heading h3 span.http_method a { background-color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.options div.heading ul.options li { border-right: 1px solid #dddddd; border-right-color: #c3d9ec; color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.options div.heading ul.options li a { color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.options div.content { background-color: #ebf3f9; border: 1px solid #c3d9ec; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.options div.content h4 { color: #0f6ab4; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.options div.content div.sandbox_header a { color: #6fa5d2; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.content, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.content, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.content, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.content, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.content, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.content { border-top: none; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.heading ul.options li:last-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.heading ul.options li:last-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.heading ul.options li:last-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.heading ul.options li:last-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.heading ul.options li:last-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.heading ul.options li:last-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.get div.heading ul.options li.last, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.post div.heading ul.options li.last, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.head div.heading ul.options li.last, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.put div.heading ul.options li.last, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.patch div.heading ul.options li.last, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations li.operation.delete div.heading ul.options li.last { padding-right: 0; border-right: none; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations ul.options li a:hover, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations ul.options li a:active, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations ul.options li a.active { text-decoration: underline; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations ul.options li:first-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations ul.options li.first { padding-left: 0; } .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations:first-child, .swagger-section .swagger-ui-wrap ul#resources li.resource ul.endpoints li.endpoint ul.operations.first { padding-left: 0; } .swagger-section .swagger-ui-wrap p#colophon { margin: 0 15px 40px 15px; padding: 10px 0; font-size: 0.8em; border-top: 1px solid #dddddd; font-family: "Droid Sans", sans-serif; color: #999999; font-style: italic; } .swagger-section .swagger-ui-wrap p#colophon a { text-decoration: none; color: #547f00; } .swagger-section .swagger-ui-wrap h3 { color: black; font-size: 1.1em; padding: 10px 0 10px 0; } .swagger-section .swagger-ui-wrap .markdown ol, .swagger-section .swagger-ui-wrap .markdown ul { font-family: "Droid Sans", sans-serif; margin: 5px 0 10px; padding: 0 0 0 18px; list-style-type: disc; } .swagger-section .swagger-ui-wrap form.form_box { background-color: #ebf3f9; border: 1px solid #c3d9ec; padding: 10px; } .swagger-section .swagger-ui-wrap form.form_box label { color: #0f6ab4 !important; } .swagger-section .swagger-ui-wrap form.form_box input[type=submit] { display: block; padding: 10px; } .swagger-section .swagger-ui-wrap form.form_box p.weak { font-size: 0.8em; } .swagger-section .swagger-ui-wrap form.form_box p { font-size: 0.9em; padding: 0 0 15px; color: #7e7b6d; } .swagger-section .swagger-ui-wrap form.form_box p a { color: #646257; } .swagger-section .swagger-ui-wrap form.form_box p strong { color: black; } .swagger-section .swagger-ui-wrap .operation-status td.markdown > p:last-child { padding-bottom: 0; } .swagger-section .title { font-style: bold; } .swagger-section .secondary_form { display: none; } .swagger-section .main_image { display: block; margin-left: auto; margin-right: auto; } .swagger-section .oauth_body { margin-left: 100px; margin-right: 100px; } .swagger-section .oauth_submit { text-align: center; display: inline-block; } .swagger-section .authorize-wrapper { margin: 15px 0 10px; } .swagger-section .authorize-wrapper_operation { float: right; } .swagger-section .authorize__btn:hover { text-decoration: underline; cursor: pointer; } .swagger-section .authorize__btn_operation:hover .authorize-scopes { display: block; } .swagger-section .authorize-scopes { position: absolute; margin-top: 20px; background: #FFF; border: 1px solid #ccc; border-radius: 5px; display: none; font-size: 13px; max-width: 300px; line-height: 30px; color: black; padding: 5px; } .swagger-section .authorize-scopes .authorize__scope { text-decoration: none; } .swagger-section .authorize__btn_operation { height: 18px; vertical-align: middle; display: inline-block; background: url(../images/explorer_icons.png) no-repeat; } .swagger-section .authorize__btn_operation_login { background-position: 0 0; width: 18px; margin-top: -6px; margin-left: 4px; } .swagger-section .authorize__btn_operation_logout { background-position: -30px 0; width: 18px; margin-top: -6px; margin-left: 4px; } .swagger-section #auth_container { color: #fff; display: inline-block; border: none; padding: 5px; width: 87px; height: 13px; } .swagger-section #auth_container .authorize__btn { color: #fff; } .swagger-section .auth_container { padding: 0 0 10px; margin-bottom: 5px; border-bottom: solid 1px #CCC; font-size: 0.9em; } .swagger-section .auth_container .auth__title { color: #547f00; font-size: 1.2em; } .swagger-section .auth_container .basic_auth__label { display: inline-block; width: 60px; } .swagger-section .auth_container .auth__description { color: #999999; margin-bottom: 5px; } .swagger-section .auth_container .auth__button { margin-top: 10px; height: 30px; } .swagger-section .auth_container .key_auth__field { margin: 5px 0; } .swagger-section .auth_container .key_auth__label { display: inline-block; width: 60px; } .swagger-section .api-popup-dialog { position: absolute; display: none; } .swagger-section .api-popup-dialog-wrapper { z-index: 1000; width: 500px; background: #FFF; padding: 20px; border: 1px solid #ccc; border-radius: 5px; font-size: 13px; color: #777; position: fixed; top: 50%; left: 50%; transform: translate(-50%, -50%); } .swagger-section .api-popup-dialog-shadow { position: fixed; top: 0; left: 0; width: 100%; height: 100%; opacity: 0.2; background-color: gray; z-index: 900; } .swagger-section .api-popup-dialog .api-popup-title { font-size: 24px; padding: 10px 0; } .swagger-section .api-popup-dialog .api-popup-title { font-size: 24px; padding: 10px 0; } .swagger-section .api-popup-dialog .error-msg { padding-left: 5px; padding-bottom: 5px; } .swagger-section .api-popup-dialog .api-popup-content { max-height: 500px; overflow-y: auto; } .swagger-section .api-popup-dialog .api-popup-authbtn { height: 30px; } .swagger-section .api-popup-dialog .api-popup-cancel { height: 30px; } .swagger-section .api-popup-scopes { padding: 10px 20px; } .swagger-section .api-popup-scopes li { padding: 5px 0; line-height: 20px; } .swagger-section .api-popup-scopes li input { position: relative; top: 2px; } .swagger-section .api-popup-scopes .api-scope-desc { padding-left: 20px; font-style: italic; } .swagger-section .api-popup-actions { padding-top: 10px; } .swagger-section .access { float: right; } .swagger-section .auth { float: right; } .swagger-section .api-ic { height: 18px; vertical-align: middle; display: inline-block; background: url(../images/explorer_icons.png) no-repeat; } .swagger-section .api-ic .api_information_panel { position: relative; margin-top: 20px; margin-left: -5px; background: #FFF; border: 1px solid #ccc; border-radius: 5px; display: none; font-size: 13px; max-width: 300px; line-height: 30px; color: black; padding: 5px; } .swagger-section .api-ic .api_information_panel p .api-msg-enabled { color: green; } .swagger-section .api-ic .api_information_panel p .api-msg-disabled { color: red; } .swagger-section .api-ic:hover .api_information_panel { position: absolute; display: block; } .swagger-section .ic-info { background-position: 0 0; width: 18px; margin-top: -6px; margin-left: 4px; } .swagger-section .ic-warning { background-position: -60px 0; width: 18px; margin-top: -6px; margin-left: 4px; } .swagger-section .ic-error { background-position: -30px 0; width: 18px; margin-top: -6px; margin-left: 4px; } .swagger-section .ic-off { background-position: -90px 0; width: 58px; margin-top: -4px; cursor: pointer; } .swagger-section .ic-on { background-position: -160px 0; width: 58px; margin-top: -4px; cursor: pointer; } .swagger-section #header { background-color: #89bf04; padding: 9px 14px 19px 14px; height: 23px; min-width: 775px; } .swagger-section #input_baseUrl { width: 400px; } .swagger-section #api_selector { display: block; clear: none; float: right; } .swagger-section #api_selector .input { display: inline-block; clear: none; margin: 0 10px 0 0; } .swagger-section #api_selector input { font-size: 0.9em; padding: 3px; margin: 0; } .swagger-section #input_apiKey { width: 200px; } .swagger-section #explore, .swagger-section #auth_container .authorize__btn { display: block; text-decoration: none; font-weight: bold; padding: 6px 8px; font-size: 0.9em; color: white; background-color: #547f00; -moz-border-radius: 4px; -webkit-border-radius: 4px; -o-border-radius: 4px; -ms-border-radius: 4px; -khtml-border-radius: 4px; border-radius: 4px; } .swagger-section #explore:hover, .swagger-section #auth_container .authorize__btn:hover { background-color: #547f00; } .swagger-section #header #logo { font-size: 1.5em; font-weight: bold; text-decoration: none; color: white; } .swagger-section #header #logo .logo__img { display: block; float: left; margin-top: 2px; } .swagger-section #header #logo .logo__title { display: inline-block; padding: 5px 0 0 10px; } .swagger-section #content_message { margin: 10px 15px; font-style: italic; color: #999999; } .swagger-section #message-bar { min-height: 30px; text-align: center; padding-top: 10px; } .swagger-section .swagger-collapse:before { content: "-"; } .swagger-section .swagger-expand:before { content: "+"; } .swagger-section .error { outline-color: #cc0000; background-color: #f2dede; } ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/css/style.css ================================================ .swagger-section #header a#logo { font-size: 1.5em; font-weight: bold; text-decoration: none; background: transparent url(../images/logo.png) no-repeat left center; padding: 20px 0 20px 40px; } #text-head { font-size: 80px; font-family: 'Roboto', sans-serif; color: #ffffff; float: right; margin-right: 20%; } .navbar-fixed-top .navbar-nav { height: auto; } .navbar-fixed-top .navbar-brand { height: auto; } .navbar-header { height: auto; } .navbar-inverse { background-color: #000; border-color: #000; } #navbar-brand { margin-left: 20%; } .navtext { font-size: 10px; } .h1, h1 { font-size: 60px; } .navbar-default .navbar-header .navbar-brand { color: #a2dfee; } /* tag titles */ .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading h2 a { color: #393939; font-family: 'Arvo', serif; font-size: 1.5em; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading h2 a:hover { color: black; } .swagger-section .swagger-ui-wrap ul#resources li.resource div.heading h2 { color: #525252; padding-left: 0px; display: block; clear: none; float: left; font-family: 'Arvo', serif; font-weight: bold; } .navbar-default .navbar-collapse, .navbar-default .navbar-form { border-color: #0A0A0A; } .container1 { width: 1500px; margin: auto; margin-top: 0; background-image: url('../images/shield.png'); background-repeat: no-repeat; background-position: -40px -20px; margin-bottom: 210px; } .container-inner { width: 1200px; margin: auto; background-color: rgba(223, 227, 228, 0.75); padding-bottom: 40px; padding-top: 40px; border-radius: 15px; } .header-content { padding: 0; width: 1000px; } .title1 { font-size: 80px; font-family: 'Vollkorn', serif; color: #404040; text-align: center; padding-top: 40px; padding-bottom: 100px; } #icon { margin-top: -18px; } .subtext { font-size: 25px; font-style: italic; color: #08b; text-align: right; padding-right: 250px; } .bg-primary { background-color: #00468b; } .navbar-default .nav > li > a, .navbar-default .nav > li > a:focus { color: #08b; } .navbar-default .nav > li > a, .navbar-default .nav > li > a:hover { color: #08b; } .navbar-default .nav > li > a, .navbar-default .nav > li > a:focus:hover { color: #08b; } .text-faded { font-size: 25px; font-family: 'Vollkorn', serif; } .section-heading { font-family: 'Vollkorn', serif; font-size: 45px; padding-bottom: 10px; } hr { border-color: #00468b; padding-bottom: 10px; } .description { margin-top: 20px; padding-bottom: 200px; } .description li { font-family: 'Vollkorn', serif; font-size: 25px; color: #525252; margin-left: 28%; padding-top: 5px; } .gap { margin-top: 200px; } .troubleshootingtext { color: rgba(255, 255, 255, 0.7); padding-left: 30%; } .troubleshootingtext li { list-style-type: circle; font-size: 25px; padding-bottom: 5px; } .overlay { position: absolute; top: 0; left: 0; width: 100%; height: 100%; z-index: 1000; } .block.response_body.json:hover { cursor: pointer; } .backdrop { color: blue; } #myModal { height: 100%; } .modal-backdrop { bottom: 0; position: fixed; } .curl { padding: 10px; font-family: "Anonymous Pro", "Menlo", "Consolas", "Bitstream Vera Sans Mono", "Courier New", monospace; font-size: 0.9em; max-height: 400px; margin-top: 5px; overflow-y: auto; background-color: #fcf6db; border: 1px solid #e5e0c6; border-radius: 4px; } .curl_title { font-size: 1.1em; margin: 0; padding: 15px 0 5px; font-family: 'Open Sans', 'Helvetica Neue', Arial, sans-serif; font-weight: 500; line-height: 1.1; } .footer { display: none; } .swagger-section .swagger-ui-wrap h2 { padding: 0; } h2 { margin: 0; margin-bottom: 5px; } .markdown p { font-size: 15px; font-family: 'Arvo', serif; } .swagger-section .swagger-ui-wrap .code { font-size: 15px; font-family: 'Arvo', serif; } .swagger-section .swagger-ui-wrap b { font-family: 'Arvo', serif; } #signin:hover { cursor: pointer; } .dropdown-menu { padding: 15px; } .navbar-right .dropdown-menu { left: 0; right: auto; } #signinbutton { width: 100%; height: 32px; font-size: 13px; font-weight: bold; color: #08b; } .navbar-default .nav > li .details { color: #000000; text-transform: none; font-size: 15px; font-weight: normal; font-family: 'Open Sans', sans-serif; font-style: italic; line-height: 20px; top: -2px; } .navbar-default .nav > li .details:hover { color: black; } #signout { width: 100%; height: 32px; font-size: 13px; font-weight: bold; color: #08b; } ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/css/typography.css ================================================ /* Google Font's Droid Sans */ @font-face { font-family: 'Droid Sans'; font-style: normal; font-weight: 400; src: local('Droid Sans'), local('DroidSans'), url('../fonts/DroidSans.ttf') format('truetype'); } /* Google Font's Droid Sans Bold */ @font-face { font-family: 'Droid Sans'; font-style: normal; font-weight: 700; src: local('Droid Sans Bold'), local('DroidSans-Bold'), url('../fonts/DroidSans-Bold.ttf') format('truetype'); } ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/index.html ================================================ Swagger UI
 
================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lang/ca.js ================================================ 'use strict'; /* jshint quotmark: double */ window.SwaggerTranslator.learn({ "Warning: Deprecated":"Advertència: Obsolet", "Implementation Notes":"Notes d'implementació", "Response Class":"Classe de la Resposta", "Status":"Estatus", "Parameters":"Paràmetres", "Parameter":"Paràmetre", "Value":"Valor", "Description":"Descripció", "Parameter Type":"Tipus del Paràmetre", "Data Type":"Tipus de la Dada", "Response Messages":"Missatges de la Resposta", "HTTP Status Code":"Codi d'Estatus HTTP", "Reason":"Raó", "Response Model":"Model de la Resposta", "Request URL":"URL de la Sol·licitud", "Response Body":"Cos de la Resposta", "Response Code":"Codi de la Resposta", "Response Headers":"Capçaleres de la Resposta", "Hide Response":"Amagar Resposta", "Try it out!":"Prova-ho!", "Show/Hide":"Mostrar/Amagar", "List Operations":"Llista Operacions", "Expand Operations":"Expandir Operacions", "Raw":"Cru", "can't parse JSON. Raw result":"no puc analitzar el JSON. Resultat cru", "Example Value":"Valor d'Exemple", "Model Schema":"Esquema del Model", "Model":"Model", "apply":"aplicar", "Username":"Nom d'usuari", "Password":"Contrasenya", "Terms of service":"Termes del servei", "Created by":"Creat per", "See more at":"Veure més en", "Contact the developer":"Contactar amb el desenvolupador", "api version":"versió de la api", "Response Content Type":"Tipus de Contingut de la Resposta", "fetching resource":"recollint recurs", "fetching resource list":"recollins llista de recursos", "Explore":"Explorant", "Show Swagger Petstore Example Apis":"Mostrar API d'Exemple Swagger Petstore", "Can't read from server. It may not have the appropriate access-control-origin settings.":"No es pot llegir del servidor. Potser no teniu la configuració de control d'accés apropiada.", "Please specify the protocol for":"Si us plau, especifiqueu el protocol per a", "Can't read swagger JSON from":"No es pot llegir el JSON de swagger des de", "Finished Loading Resource Information. Rendering Swagger UI":"Finalitzada la càrrega del recurs informatiu. Renderitzant Swagger UI", "Unable to read api":"No es pot llegir l'api", "from path":"des de la ruta", "server returned":"el servidor ha retornat" }); ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lang/el.js ================================================ 'use strict'; /* jshint quotmark: double */ window.SwaggerTranslator.learn({ "Warning: Deprecated":"Προειδοποίηση: Έχει αποσυρθεί", "Implementation Notes":"Σημειώσεις Υλοποίησης", "Response Class":"Απόκριση", "Status":"Κατάσταση", "Parameters":"Παράμετροι", "Parameter":"Παράμετρος", "Value":"Τιμή", "Description":"Περιγραφή", "Parameter Type":"Τύπος Παραμέτρου", "Data Type":"Τύπος Δεδομένων", "Response Messages":"Μηνύματα Απόκρισης", "HTTP Status Code":"Κωδικός Κατάστασης HTTP", "Reason":"Αιτιολογία", "Response Model":"Μοντέλο Απόκρισης", "Request URL":"URL Αιτήματος", "Response Body":"Σώμα Απόκρισης", "Response Code":"Κωδικός Απόκρισης", "Response Headers":"Επικεφαλίδες Απόκρισης", "Hide Response":"Απόκρυψη Απόκρισης", "Headers":"Επικεφαλίδες", "Try it out!":"Δοκιμάστε το!", "Show/Hide":"Εμφάνιση/Απόκρυψη", "List Operations":"Λίστα Λειτουργιών", "Expand Operations":"Ανάπτυξη Λειτουργιών", "Raw":"Ακατέργαστο", "can't parse JSON. Raw result":"αδυναμία ανάλυσης JSON. Ακατέργαστο αποτέλεσμα", "Example Value":"Παράδειγμα Τιμής", "Model Schema":"Σχήμα Μοντέλου", "Model":"Μοντέλο", "Click to set as parameter value":"Πατήστε για να θέσετε τιμή παραμέτρου", "apply":"εφαρμογή", "Username":"Όνομα χρήση", "Password":"Κωδικός πρόσβασης", "Terms of service":"Όροι χρήσης", "Created by":"Δημιουργήθηκε από", "See more at":"Δείτε περισσότερα στο", "Contact the developer":"Επικοινωνήστε με τον προγραμματιστή", "api version":"έκδοση api", "Response Content Type":"Τύπος Περιεχομένου Απόκρισης", "Parameter content type:":"Τύπος περιεχομένου παραμέτρου:", "fetching resource":"παραλαβή πόρου", "fetching resource list":"παραλαβή λίστας πόρων", "Explore":"Εξερεύνηση", "Show Swagger Petstore Example Apis":"Εμφάνιση Api Δειγμάτων Petstore του Swagger", "Can't read from server. It may not have the appropriate access-control-origin settings.":"Αδυναμία ανάγνωσης από τον εξυπηρετητή. Μπορεί να μην έχει κατάλληλες ρυθμίσεις για access-control-origin.", "Please specify the protocol for":"Παρακαλώ προσδιορίστε το πρωτόκολλο για", "Can't read swagger JSON from":"Αδυναμία ανάγνωσης swagger JSON από", "Finished Loading Resource Information. Rendering Swagger UI":"Ολοκλήρωση Φόρτωσης Πληροφορικών Πόρου. Παρουσίαση Swagger UI", "Unable to read api":"Αδυναμία ανάγνωσης api", "from path":"από το μονοπάτι", "server returned":"ο εξυπηρετηρής επέστρεψε" }); ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lang/en.js ================================================ 'use strict'; /* jshint quotmark: double */ window.SwaggerTranslator.learn({ "Warning: Deprecated":"Warning: Deprecated", "Implementation Notes":"Implementation Notes", "Response Class":"Response Class", "Status":"Status", "Parameters":"Parameters", "Parameter":"Parameter", "Value":"Value", "Description":"Description", "Parameter Type":"Parameter Type", "Data Type":"Data Type", "Response Messages":"Response Messages", "HTTP Status Code":"HTTP Status Code", "Reason":"Reason", "Response Model":"Response Model", "Request URL":"Request URL", "Response Body":"Response Body", "Response Code":"Response Code", "Response Headers":"Response Headers", "Hide Response":"Hide Response", "Headers":"Headers", "Try it out!":"Try it out!", "Show/Hide":"Show/Hide", "List Operations":"List Operations", "Expand Operations":"Expand Operations", "Raw":"Raw", "can't parse JSON. Raw result":"can't parse JSON. Raw result", "Example Value":"Example Value", "Model Schema":"Model Schema", "Model":"Model", "Click to set as parameter value":"Click to set as parameter value", "apply":"apply", "Username":"Username", "Password":"Password", "Terms of service":"Terms of service", "Created by":"Created by", "See more at":"See more at", "Contact the developer":"Contact the developer", "api version":"api version", "Response Content Type":"Response Content Type", "Parameter content type:":"Parameter content type:", "fetching resource":"fetching resource", "fetching resource list":"fetching resource list", "Explore":"Explore", "Show Swagger Petstore Example Apis":"Show Swagger Petstore Example Apis", "Can't read from server. It may not have the appropriate access-control-origin settings.":"Can't read from server. It may not have the appropriate access-control-origin settings.", "Please specify the protocol for":"Please specify the protocol for", "Can't read swagger JSON from":"Can't read swagger JSON from", "Finished Loading Resource Information. Rendering Swagger UI":"Finished Loading Resource Information. Rendering Swagger UI", "Unable to read api":"Unable to read api", "from path":"from path", "server returned":"server returned" }); ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lang/es.js ================================================ 'use strict'; /* jshint quotmark: double */ window.SwaggerTranslator.learn({ "Warning: Deprecated":"Advertencia: Obsoleto", "Implementation Notes":"Notas de implementación", "Response Class":"Clase de la Respuesta", "Status":"Status", "Parameters":"Parámetros", "Parameter":"Parámetro", "Value":"Valor", "Description":"Descripción", "Parameter Type":"Tipo del Parámetro", "Data Type":"Tipo del Dato", "Response Messages":"Mensajes de la Respuesta", "HTTP Status Code":"Código de Status HTTP", "Reason":"Razón", "Response Model":"Modelo de la Respuesta", "Request URL":"URL de la Solicitud", "Response Body":"Cuerpo de la Respuesta", "Response Code":"Código de la Respuesta", "Response Headers":"Encabezados de la Respuesta", "Hide Response":"Ocultar Respuesta", "Try it out!":"Pruébalo!", "Show/Hide":"Mostrar/Ocultar", "List Operations":"Listar Operaciones", "Expand Operations":"Expandir Operaciones", "Raw":"Crudo", "can't parse JSON. Raw result":"no puede parsear el JSON. Resultado crudo", "Example Value":"Valor de Ejemplo", "Model Schema":"Esquema del Modelo", "Model":"Modelo", "apply":"aplicar", "Username":"Nombre de usuario", "Password":"Contraseña", "Terms of service":"Términos de Servicio", "Created by":"Creado por", "See more at":"Ver más en", "Contact the developer":"Contactar al desarrollador", "api version":"versión de la api", "Response Content Type":"Tipo de Contenido (Content Type) de la Respuesta", "fetching resource":"buscando recurso", "fetching resource list":"buscando lista del recurso", "Explore":"Explorar", "Show Swagger Petstore Example Apis":"Mostrar Api Ejemplo de Swagger Petstore", "Can't read from server. It may not have the appropriate access-control-origin settings.":"No se puede leer del servidor. Tal vez no tiene la configuración de control de acceso de origen (access-control-origin) apropiado.", "Please specify the protocol for":"Por favor, especificar el protocola para", "Can't read swagger JSON from":"No se puede leer el JSON de swagger desde", "Finished Loading Resource Information. Rendering Swagger UI":"Finalizada la carga del recurso de Información. Mostrando Swagger UI", "Unable to read api":"No se puede leer la api", "from path":"desde ruta", "server returned":"el servidor retornó" }); ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lang/fr.js ================================================ 'use strict'; /* jshint quotmark: double */ window.SwaggerTranslator.learn({ "Warning: Deprecated":"Avertissement : Obsolète", "Implementation Notes":"Notes d'implémentation", "Response Class":"Classe de la réponse", "Status":"Statut", "Parameters":"Paramètres", "Parameter":"Paramètre", "Value":"Valeur", "Description":"Description", "Parameter Type":"Type du paramètre", "Data Type":"Type de données", "Response Messages":"Messages de la réponse", "HTTP Status Code":"Code de statut HTTP", "Reason":"Raison", "Response Model":"Modèle de réponse", "Request URL":"URL appelée", "Response Body":"Corps de la réponse", "Response Code":"Code de la réponse", "Response Headers":"En-têtes de la réponse", "Hide Response":"Cacher la réponse", "Headers":"En-têtes", "Try it out!":"Testez !", "Show/Hide":"Afficher/Masquer", "List Operations":"Liste des opérations", "Expand Operations":"Développer les opérations", "Raw":"Brut", "can't parse JSON. Raw result":"impossible de décoder le JSON. Résultat brut", "Example Value":"Exemple la valeur", "Model Schema":"Définition du modèle", "Model":"Modèle", "apply":"appliquer", "Username":"Nom d'utilisateur", "Password":"Mot de passe", "Terms of service":"Conditions de service", "Created by":"Créé par", "See more at":"Voir plus sur", "Contact the developer":"Contacter le développeur", "api version":"version de l'api", "Response Content Type":"Content Type de la réponse", "fetching resource":"récupération de la ressource", "fetching resource list":"récupération de la liste de ressources", "Explore":"Explorer", "Show Swagger Petstore Example Apis":"Montrer les Apis de l'exemple Petstore de Swagger", "Can't read from server. It may not have the appropriate access-control-origin settings.":"Impossible de lire à partir du serveur. Il se peut que les réglages access-control-origin ne soient pas appropriés.", "Please specify the protocol for":"Veuillez spécifier un protocole pour", "Can't read swagger JSON from":"Impossible de lire le JSON swagger à partir de", "Finished Loading Resource Information. Rendering Swagger UI":"Chargement des informations terminé. Affichage de Swagger UI", "Unable to read api":"Impossible de lire l'api", "from path":"à partir du chemin", "server returned":"réponse du serveur" }); ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lang/geo.js ================================================ 'use strict'; /* jshint quotmark: double */ window.SwaggerTranslator.learn({ "Warning: Deprecated":"ყურადღება: აღარ გამოიყენება", "Implementation Notes":"იმპლემენტაციის აღწერა", "Response Class":"რესპონს კლასი", "Status":"სტატუსი", "Parameters":"პარამეტრები", "Parameter":"პარამეტრი", "Value":"მნიშვნელობა", "Description":"აღწერა", "Parameter Type":"პარამეტრის ტიპი", "Data Type":"მონაცემის ტიპი", "Response Messages":"პასუხი", "HTTP Status Code":"HTTP სტატუსი", "Reason":"მიზეზი", "Response Model":"რესპონს მოდელი", "Request URL":"მოთხოვნის URL", "Response Body":"პასუხის სხეული", "Response Code":"პასუხის კოდი", "Response Headers":"პასუხის ჰედერები", "Hide Response":"დამალე პასუხი", "Headers":"ჰედერები", "Try it out!":"ცადე !", "Show/Hide":"გამოჩენა/დამალვა", "List Operations":"ოპერაციების სია", "Expand Operations":"ოპერაციები ვრცლად", "Raw":"ნედლი", "can't parse JSON. Raw result":"JSON-ის დამუშავება ვერ მოხერხდა. ნედლი პასუხი", "Example Value":"მაგალითი", "Model Schema":"მოდელის სტრუქტურა", "Model":"მოდელი", "Click to set as parameter value":"პარამეტრისთვის მნიშვნელობის მისანიჭებლად, დააკლიკე", "apply":"გამოყენება", "Username":"მოხმარებელი", "Password":"პაროლი", "Terms of service":"მომსახურების პირობები", "Created by":"შექმნა", "See more at":"ნახე ვრცლად", "Contact the developer":"დაუკავშირდი დეველოპერს", "api version":"api ვერსია", "Response Content Type":"პასუხის კონტენტის ტიპი", "Parameter content type:":"პარამეტრის კონტენტის ტიპი:", "fetching resource":"რესურსების მიღება", "fetching resource list":"რესურსების სიის მიღება", "Explore":"ნახვა", "Show Swagger Petstore Example Apis":"ნახე Swagger Petstore სამაგალითო Api", "Can't read from server. It may not have the appropriate access-control-origin settings.":"სერვერთან დაკავშირება ვერ ხერხდება. შეამოწმეთ access-control-origin.", "Please specify the protocol for":"მიუთითეთ პროტოკოლი", "Can't read swagger JSON from":"swagger JSON წაკითხვა ვერ მოხერხდა", "Finished Loading Resource Information. Rendering Swagger UI":"რესურსების ჩატვირთვა სრულდება. Swagger UI რენდერდება", "Unable to read api":"api წაკითხვა ვერ მოხერხდა", "from path":"მისამართიდან", "server returned":"სერვერმა დააბრუნა" }); ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lang/it.js ================================================ 'use strict'; /* jshint quotmark: double */ window.SwaggerTranslator.learn({ "Warning: Deprecated":"Attenzione: Deprecato", "Implementation Notes":"Note di implementazione", "Response Class":"Classe della risposta", "Status":"Stato", "Parameters":"Parametri", "Parameter":"Parametro", "Value":"Valore", "Description":"Descrizione", "Parameter Type":"Tipo di parametro", "Data Type":"Tipo di dato", "Response Messages":"Messaggi della risposta", "HTTP Status Code":"Codice stato HTTP", "Reason":"Motivo", "Response Model":"Modello di risposta", "Request URL":"URL della richiesta", "Response Body":"Corpo della risposta", "Response Code":"Oggetto della risposta", "Response Headers":"Intestazioni della risposta", "Hide Response":"Nascondi risposta", "Try it out!":"Provalo!", "Show/Hide":"Mostra/Nascondi", "List Operations":"Mostra operazioni", "Expand Operations":"Espandi operazioni", "Raw":"Grezzo (raw)", "can't parse JSON. Raw result":"non è possibile parsare il JSON. Risultato grezzo (raw).", "Model Schema":"Schema del modello", "Model":"Modello", "apply":"applica", "Username":"Nome utente", "Password":"Password", "Terms of service":"Condizioni del servizio", "Created by":"Creato da", "See more at":"Informazioni aggiuntive:", "Contact the developer":"Contatta lo sviluppatore", "api version":"versione api", "Response Content Type":"Tipo di contenuto (content type) della risposta", "fetching resource":"recuperando la risorsa", "fetching resource list":"recuperando lista risorse", "Explore":"Esplora", "Show Swagger Petstore Example Apis":"Mostra le api di esempio di Swagger Petstore", "Can't read from server. It may not have the appropriate access-control-origin settings.":"Non è possibile leggere dal server. Potrebbe non avere le impostazioni di controllo accesso origine (access-control-origin) appropriate.", "Please specify the protocol for":"Si prega di specificare il protocollo per", "Can't read swagger JSON from":"Impossibile leggere JSON swagger da:", "Finished Loading Resource Information. Rendering Swagger UI":"Lettura informazioni risorse termianta. Swagger UI viene mostrata", "Unable to read api":"Impossibile leggere la api", "from path":"da cartella", "server returned":"il server ha restituito" }); ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lang/ja.js ================================================ 'use strict'; /* jshint quotmark: double */ window.SwaggerTranslator.learn({ "Warning: Deprecated":"警告: 廃止予定", "Implementation Notes":"実装メモ", "Response Class":"レスポンスクラス", "Status":"ステータス", "Parameters":"パラメータ群", "Parameter":"パラメータ", "Value":"値", "Description":"説明", "Parameter Type":"パラメータタイプ", "Data Type":"データタイプ", "Response Messages":"レスポンスメッセージ", "HTTP Status Code":"HTTPステータスコード", "Reason":"理由", "Response Model":"レスポンスモデル", "Request URL":"リクエストURL", "Response Body":"レスポンスボディ", "Response Code":"レスポンスコード", "Response Headers":"レスポンスヘッダ", "Hide Response":"レスポンスを隠す", "Headers":"ヘッダ", "Try it out!":"実際に実行!", "Show/Hide":"表示/非表示", "List Operations":"操作一覧", "Expand Operations":"操作の展開", "Raw":"未加工", "can't parse JSON. Raw result":"JSONへ解釈できません. 未加工の結果", "Example Value":"値の例", "Model Schema":"モデルスキーマ", "Model":"モデル", "Click to set as parameter value":"パラメータ値と設定するにはクリック", "apply":"実行", "Username":"ユーザ名", "Password":"パスワード", "Terms of service":"サービス利用規約", "Created by":"Created by", "See more at":"詳細を見る", "Contact the developer":"開発者に連絡", "api version":"APIバージョン", "Response Content Type":"レスポンス コンテンツタイプ", "Parameter content type:":"パラメータコンテンツタイプ:", "fetching resource":"リソースの取得", "fetching resource list":"リソース一覧の取得", "Explore":"調査", "Show Swagger Petstore Example Apis":"SwaggerペットストアAPIの表示", "Can't read from server. It may not have the appropriate access-control-origin settings.":"サーバから読み込めません. 適切なaccess-control-origin設定を持っていない可能性があります.", "Please specify the protocol for":"プロトコルを指定してください", "Can't read swagger JSON from":"次からswagger JSONを読み込めません", "Finished Loading Resource Information. Rendering Swagger UI":"リソース情報の読み込みが完了しました. Swagger UIを描画しています", "Unable to read api":"APIを読み込めません", "from path":"次のパスから", "server returned":"サーバからの返答" }); ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lang/ko-kr.js ================================================ 'use strict'; /* jshint quotmark: double */ window.SwaggerTranslator.learn({ "Warning: Deprecated":"경고：폐기예정됨", "Implementation Notes":"구현 노트", "Response Class":"응답 클래스", "Status":"상태", "Parameters":"매개변수들", "Parameter":"매개변수", "Value":"값", "Description":"설명", "Parameter Type":"매개변수 타입", "Data Type":"데이터 타입", "Response Messages":"응답 메세지", "HTTP Status Code":"HTTP 상태 코드", "Reason":"원인", "Response Model":"응답 모델", "Request URL":"요청 URL", "Response Body":"응답 본문", "Response Code":"응답 코드", "Response Headers":"응답 헤더", "Hide Response":"응답 숨기기", "Headers":"헤더", "Try it out!":"써보기！", "Show/Hide":"보이기/숨기기", "List Operations":"목록 작업", "Expand Operations":"전개 작업", "Raw":"원본", "can't parse JSON. Raw result":"JSON을 파싱할수 없음. 원본결과:", "Model Schema":"모델 스키마", "Model":"모델", "apply":"적용", "Username":"사용자 이름", "Password":"암호", "Terms of service":"이용약관", "Created by":"작성자", "See more at":"추가정보：", "Contact the developer":"개발자에게 문의", "api version":"api버전", "Response Content Type":"응답Content Type", "fetching resource":"리소스 가져오기", "fetching resource list":"리소스 목록 가져오기", "Explore":"탐색", "Show Swagger Petstore Example Apis":"Swagger Petstore 예제 보기", "Can't read from server. It may not have the appropriate access-control-origin settings.":"서버로부터 읽어들일수 없습니다. access-control-origin 설정이 올바르지 않을수 있습니다.", "Please specify the protocol for":"다음을 위한 프로토콜을 정하세요", "Can't read swagger JSON from":"swagger JSON 을 다음으로 부터 읽을수 없습니다", "Finished Loading Resource Information. Rendering Swagger UI":"리소스 정보 불러오기 완료. Swagger UI 랜더링", "Unable to read api":"api를 읽을 수 없습니다.", "from path":"다음 경로로 부터", "server returned":"서버 응답함." }); ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lang/pl.js ================================================ 'use strict'; /* jshint quotmark: double */ window.SwaggerTranslator.learn({ "Warning: Deprecated":"Uwaga: Wycofane", "Implementation Notes":"Uwagi Implementacji", "Response Class":"Klasa Odpowiedzi", "Status":"Status", "Parameters":"Parametry", "Parameter":"Parametr", "Value":"Wartość", "Description":"Opis", "Parameter Type":"Typ Parametru", "Data Type":"Typ Danych", "Response Messages":"Wiadomości Odpowiedzi", "HTTP Status Code":"Kod Statusu HTTP", "Reason":"Przyczyna", "Response Model":"Model Odpowiedzi", "Request URL":"URL Wywołania", "Response Body":"Treść Odpowiedzi", "Response Code":"Kod Odpowiedzi", "Response Headers":"Nagłówki Odpowiedzi", "Hide Response":"Ukryj Odpowiedź", "Headers":"Nagłówki", "Try it out!":"Wypróbuj!", "Show/Hide":"Pokaż/Ukryj", "List Operations":"Lista Operacji", "Expand Operations":"Rozwiń Operacje", "Raw":"Nieprzetworzone", "can't parse JSON. Raw result":"nie można przetworzyć pliku JSON. Nieprzetworzone dane", "Model Schema":"Schemat Modelu", "Model":"Model", "apply":"użyj", "Username":"Nazwa użytkownika", "Password":"Hasło", "Terms of service":"Warunki używania", "Created by":"Utworzone przez", "See more at":"Zobacz więcej na", "Contact the developer":"Kontakt z deweloperem", "api version":"wersja api", "Response Content Type":"Typ Zasobu Odpowiedzi", "fetching resource":"ładowanie zasobu", "fetching resource list":"ładowanie listy zasobów", "Explore":"Eksploruj", "Show Swagger Petstore Example Apis":"Pokaż Przykładowe Api Swagger Petstore", "Can't read from server. It may not have the appropriate access-control-origin settings.":"Brak połączenia z serwerem. Może on nie mieć odpowiednich ustawień access-control-origin.", "Please specify the protocol for":"Proszę podać protokół dla", "Can't read swagger JSON from":"Nie można odczytać swagger JSON z", "Finished Loading Resource Information. Rendering Swagger UI":"Ukończono Ładowanie Informacji o Zasobie. Renderowanie Swagger UI", "Unable to read api":"Nie można odczytać api", "from path":"ze ścieżki", "server returned":"serwer zwrócił" }); ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lang/pt.js ================================================ 'use strict'; /* jshint quotmark: double */ window.SwaggerTranslator.learn({ "Warning: Deprecated":"Aviso: Depreciado", "Implementation Notes":"Notas de Implementação", "Response Class":"Classe de resposta", "Status":"Status", "Parameters":"Parâmetros", "Parameter":"Parâmetro", "Value":"Valor", "Description":"Descrição", "Parameter Type":"Tipo de parâmetro", "Data Type":"Tipo de dados", "Response Messages":"Mensagens de resposta", "HTTP Status Code":"Código de status HTTP", "Reason":"Razão", "Response Model":"Modelo resposta", "Request URL":"URL requisição", "Response Body":"Corpo da resposta", "Response Code":"Código da resposta", "Response Headers":"Cabeçalho da resposta", "Headers":"Cabeçalhos", "Hide Response":"Esconder resposta", "Try it out!":"Tente agora!", "Show/Hide":"Mostrar/Esconder", "List Operations":"Listar operações", "Expand Operations":"Expandir operações", "Raw":"Cru", "can't parse JSON. Raw result":"Falha ao analisar JSON. Resulto cru", "Model Schema":"Modelo esquema", "Model":"Modelo", "apply":"Aplicar", "Username":"Usuário", "Password":"Senha", "Terms of service":"Termos do serviço", "Created by":"Criado por", "See more at":"Veja mais em", "Contact the developer":"Contate o desenvolvedor", "api version":"Versão api", "Response Content Type":"Tipo de conteúdo da resposta", "fetching resource":"busca recurso", "fetching resource list":"buscando lista de recursos", "Explore":"Explorar", "Show Swagger Petstore Example Apis":"Show Swagger Petstore Example Apis", "Can't read from server. It may not have the appropriate access-control-origin settings.":"Não é possível ler do servidor. Pode não ter as apropriadas configurações access-control-origin", "Please specify the protocol for":"Por favor especifique o protocolo", "Can't read swagger JSON from":"Não é possível ler o JSON Swagger de", "Finished Loading Resource Information. Rendering Swagger UI":"Carregar informação de recurso finalizada. Renderizando Swagger UI", "Unable to read api":"Não foi possível ler api", "from path":"do caminho", "server returned":"servidor retornou" }); ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lang/ru.js ================================================ 'use strict'; /* jshint quotmark: double */ window.SwaggerTranslator.learn({ "Warning: Deprecated":"Предупреждение: Устарело", "Implementation Notes":"Заметки", "Response Class":"Пример ответа", "Status":"Статус", "Parameters":"Параметры", "Parameter":"Параметр", "Value":"Значение", "Description":"Описание", "Parameter Type":"Тип параметра", "Data Type":"Тип данных", "HTTP Status Code":"HTTP код", "Reason":"Причина", "Response Model":"Структура ответа", "Request URL":"URL запроса", "Response Body":"Тело ответа", "Response Code":"HTTP код ответа", "Response Headers":"Заголовки ответа", "Hide Response":"Спрятать ответ", "Headers":"Заголовки", "Response Messages":"Что может прийти в ответ", "Try it out!":"Попробовать!", "Show/Hide":"Показать/Скрыть", "List Operations":"Операции кратко", "Expand Operations":"Операции подробно", "Raw":"В сыром виде", "can't parse JSON. Raw result":"Не удается распарсить ответ:", "Example Value":"Пример", "Model Schema":"Структура", "Model":"Описание", "Click to set as parameter value":"Нажмите, чтобы испльзовать в качестве значения параметра", "apply":"применить", "Username":"Имя пользователя", "Password":"Пароль", "Terms of service":"Условия использования", "Created by":"Разработано", "See more at":"Еще тут", "Contact the developer":"Связаться с разработчиком", "api version":"Версия API", "Response Content Type":"Content Type ответа", "Parameter content type:":"Content Type параметра:", "fetching resource":"Получение ресурса", "fetching resource list":"Получение ресурсов", "Explore":"Показать", "Show Swagger Petstore Example Apis":"Показать примеры АПИ", "Can't read from server. It may not have the appropriate access-control-origin settings.":"Не удается получить ответ от сервера. Возможно, проблема с настройками доступа", "Please specify the protocol for":"Пожалуйста, укажите протокол для", "Can't read swagger JSON from":"Не получается прочитать swagger json из", "Finished Loading Resource Information. Rendering Swagger UI":"Загрузка информации о ресурсах завершена. Рендерим", "Unable to read api":"Не удалось прочитать api", "from path":"по адресу", "server returned":"сервер сказал" }); ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lang/tr.js ================================================ 'use strict'; /* jshint quotmark: double */ window.SwaggerTranslator.learn({ "Warning: Deprecated":"Uyarı: Deprecated", "Implementation Notes":"Gerçekleştirim Notları", "Response Class":"Dönen Sınıf", "Status":"Statü", "Parameters":"Parametreler", "Parameter":"Parametre", "Value":"Değer", "Description":"Açıklama", "Parameter Type":"Parametre Tipi", "Data Type":"Veri Tipi", "Response Messages":"Dönüş Mesajı", "HTTP Status Code":"HTTP Statü Kodu", "Reason":"Gerekçe", "Response Model":"Dönüş Modeli", "Request URL":"İstek URL", "Response Body":"Dönüş İçeriği", "Response Code":"Dönüş Kodu", "Response Headers":"Dönüş Üst Bilgileri", "Hide Response":"Dönüşü Gizle", "Headers":"Üst Bilgiler", "Try it out!":"Dene!", "Show/Hide":"Göster/Gizle", "List Operations":"Operasyonları Listele", "Expand Operations":"Operasyonları Aç", "Raw":"Ham", "can't parse JSON. Raw result":"JSON çözümlenemiyor. Ham sonuç", "Model Schema":"Model Şema", "Model":"Model", "apply":"uygula", "Username":"Kullanıcı Adı", "Password":"Parola", "Terms of service":"Servis şartları", "Created by":"Oluşturan", "See more at":"Daha fazlası için", "Contact the developer":"Geliştirici ile İletişime Geçin", "api version":"api versiyon", "Response Content Type":"Dönüş İçerik Tipi", "fetching resource":"kaynak getiriliyor", "fetching resource list":"kaynak listesi getiriliyor", "Explore":"Keşfet", "Show Swagger Petstore Example Apis":"Swagger Petstore Örnek Api'yi Gör", "Can't read from server. It may not have the appropriate access-control-origin settings.":"Sunucudan okuma yapılamıyor. Sunucu access-control-origin ayarlarınızı kontrol edin.", "Please specify the protocol for":"Lütfen istenen adres için protokol belirtiniz", "Can't read swagger JSON from":"Swagger JSON bu kaynaktan okunamıyor", "Finished Loading Resource Information. Rendering Swagger UI":"Kaynak baglantısı tamamlandı. Swagger UI gösterime hazırlanıyor", "Unable to read api":"api okunamadı", "from path":"yoldan", "server returned":"sunucuya dönüldü" }); ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lang/translator.js ================================================ 'use strict'; /** * Translator for documentation pages. * * To enable translation you should include one of language-files in your index.html * after . * For example - * * If you wish to translate some new texts you should do two things: * 1. Add a new phrase pair ("New Phrase": "New Translation") into your language file (for example lang/ru.js). It will be great if you add it in other language files too. * 2. Mark that text it templates this way New Phrase or . * The main thing here is attribute data-sw-translate. Only inner html, title-attribute and value-attribute are going to translate. * */ window.SwaggerTranslator = { _words:[], translate: function(sel) { var $this = this; sel = sel || '[data-sw-translate]'; $(sel).each(function() { $(this).html($this._tryTranslate($(this).html())); $(this).val($this._tryTranslate($(this).val())); $(this).attr('title', $this._tryTranslate($(this).attr('title'))); }); }, _tryTranslate: function(word) { return this._words[$.trim(word)] !== undefined ? this._words[$.trim(word)] : word; }, learn: function(wordsMap) { this._words = wordsMap; } }; ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lang/zh-cn.js ================================================ 'use strict'; /* jshint quotmark: double */ window.SwaggerTranslator.learn({ "Warning: Deprecated":"警告：已过时", "Implementation Notes":"实现备注", "Response Class":"响应类", "Status":"状态", "Parameters":"参数", "Parameter":"参数", "Value":"值", "Description":"描述", "Parameter Type":"参数类型", "Data Type":"数据类型", "Response Messages":"响应消息", "HTTP Status Code":"HTTP状态码", "Reason":"原因", "Response Model":"响应模型", "Request URL":"请求URL", "Response Body":"响应体", "Response Code":"响应码", "Response Headers":"响应头", "Hide Response":"隐藏响应", "Headers":"头", "Try it out!":"试一下！", "Show/Hide":"显示/隐藏", "List Operations":"显示操作", "Expand Operations":"展开操作", "Raw":"原始", "can't parse JSON. Raw result":"无法解析JSON. 原始结果", "Example Value":"示例", "Click to set as parameter value":"点击设置参数", "Model Schema":"模型架构", "Model":"模型", "apply":"应用", "Username":"用户名", "Password":"密码", "Terms of service":"服务条款", "Created by":"创建者", "See more at":"查看更多：", "Contact the developer":"联系开发者", "api version":"api版本", "Response Content Type":"响应Content Type", "Parameter content type:":"参数类型:", "fetching resource":"正在获取资源", "fetching resource list":"正在获取资源列表", "Explore":"浏览", "Show Swagger Petstore Example Apis":"显示 Swagger Petstore 示例 Apis", "Can't read from server. It may not have the appropriate access-control-origin settings.":"无法从服务器读取。可能没有正确设置access-control-origin。", "Please specify the protocol for":"请指定协议：", "Can't read swagger JSON from":"无法读取swagger JSON于", "Finished Loading Resource Information. Rendering Swagger UI":"已加载资源信息。正在渲染Swagger UI", "Unable to read api":"无法读取api", "from path":"从路径", "server returned":"服务器返回" }); ================================================ FILE: src/backend/tools/py.doc/caliopen_api_doc/swagger-ui/lib/backbone-min.js ================================================ // Backbone.js 1.1.2 (function(t,e){if(typeof define==="function"&&define.amd){define(["underscore","jquery","exports"],function(i,r,s){t.Backbone=e(t,s,i,r)})}else if(typeof exports!=="undefined"){var i=require("underscore");e(t,exports,i)}else{t.Backbone=e(t,{},t._,t.jQuery||t.Zepto||t.ender||t.$)}})(this,function(t,e,i,r){var s=t.Backbone;var n=[];var a=n.push;var o=n.slice;var h=n.splice;e.VERSION="1.1.2";e.$=r;e.noConflict=function(){t.Backbone=s;return this};e.emulateHTTP=false;e.emulateJSON=false;var u=e.Events={on:function(t,e,i){if(!c(this,"on",t,[e,i])||!e)return this;this._events||(this._events={});var r=this._events[t]||(this._events[t]=[]);r.push({callback:e,context:i,ctx:i||this});return this},once:function(t,e,r){if(!c(this,"once",t,[e,r])||!e)return this;var s=this;var n=i.once(function(){s.off(t,n);e.apply(this,arguments)});n._callback=e;return this.on(t,n,r)},off:function(t,e,r){var s,n,a,o,h,u,l,f;if(!this._events||!c(this,"off",t,[e,r]))return this;if(!t&&!e&&!r){this._events=void 0;return this}o=t?[t]:i.keys(this._events);for(h=0,u=o.length;h").attr(t);this.setElement(r,false)}else{this.setElement(i.result(this,"el"),false)}}});e.sync=function(t,r,s){var n=T[t];i.defaults(s||(s={}),{emulateHTTP:e.emulateHTTP,emulateJSON:e.emulateJSON});var a={type:n,dataType:"json"};if(!s.url){a.url=i.result(r,"url")||M()}if(s.data==null&&r&&(t==="create"||t==="update"||t==="patch")){a.contentType="application/json";a.data=JSON.stringify(s.attrs||r.toJSON(s))}if(s.emulateJSON){a.contentType="application/x-www-form-urlencoded";a.data=a.data?{model:a.data}:{}}if(s.emulateHTTP&&(n==="PUT"||n==="DELETE"||n==="PATCH")){a.type="POST";if(s.emulateJSON)a.data._method=n;var o=s.beforeSend;s.beforeSend=function(t){t.setRequestHeader("X-HTTP-Method-Override",n);if(o)return o.apply(this,arguments)}}if(a.type!=="GET"&&!s.emulateJSON){a.processData=false}if(a.type==="PATCH"&&k){a.xhr=function(){return new ActiveXObject("Microsoft.XMLHTTP")}}var h=s.xhr=e.ajax(i.extend(a,s));r.trigger("request",r,h,s);return h};var k=typeof window!=="undefined"&&!!window.ActiveXObject&&!(window.XMLHttpRequest&&(new XMLHttpRequest).dispatchEvent);var T={create:"POST",update:"PUT",patch:"PATCH","delete":"DELETE",read:"GET"};e.ajax=function(){return e.$.ajax.apply(e.$,arguments)};var $=e.Router=function(t){t||(t={});if(t.routes)this.routes=t.routes;this._bindRoutes();this.initialize.apply(this,arguments)};var S=/\((.*?)\)/g;var H=/(\(\?)?:\w+/g;var A=/\*\w+/g;var I=/[\-{}\[\]+?.,\\\^$|#\s]/g;i.extend($.prototype,u,{initialize:function(){},route:function(t,r,s){if(!i.isRegExp(t))t=this._routeToRegExp(t);if(i.isFunction(r)){s=r;r=""}if(!s)s=this[r];var n=this;e.history.route(t,function(i){var a=n._extractParameters(t,i);n.execute(s,a);n.trigger.apply(n,["route:"+r].concat(a));n.trigger("route",r,a);e.history.trigger("route",n,r,a)});return this},execute:function(t,e){if(t)t.apply(this,e)},navigate:function(t,i){e.history.navigate(t,i);return this},_bindRoutes:function(){if(!this.routes)return;this.routes=i.result(this,"routes");var t,e=i.keys(this.routes);while((t=e.pop())!=null){this.route(t,this.routes[t])}},_routeToRegExp:function(t){t=t.replace(I,"\\$&").replace(S,"(?:$1)?").replace(H,function(t,e){return e?t:"([^/?]+)"}).replace(A,"([^?]*?)");return new RegExp("^"+t+"(?:\\?([\\s\\S]*))?$")},_extractParameters:function(t,e){var r=t.exec(e).slice(1);return i.map(r,function(t,e){if(e===r.length-1)return t||null;return t?decodeURIComponent(t):null})}});var N=e.History=function(){this.handlers=[];i.bindAll(this,"checkUrl");if(typeof window!=="undefined"){this.location=window.location;this.history=window.history}};var R=/^[#\/]|\s+$/g;var O=/^\/+|\/+$/g;var P=/msie [\w.]+/;var C=/\/$/;var j=/#.*$/;N.started=false;i.extend(N.prototype,u,{interval:50,atRoot:function(){return this.location.pathname.replace(/[^\/]$/,"$&/")===this.root},getHash:function(t){var e=(t||this).location.href.match(/#(.*)$/);return e?e[1]:""},getFragment:function(t,e){if(t==null){if(this._hasPushState||!this._wantsHashChange||e){t=decodeURI(this.location.pathname+this.location.search);var i=this.root.replace(C,"");if(!t.indexOf(i))t=t.slice(i.length)}else{t=this.getHash()}}return t.replace(R,"")},start:function(t){if(N.started)throw new Error("Backbone.history has already been started");N.started=true;this.options=i.extend({root:"/"},this.options,t);this.root=this.options.root;this._wantsHashChange=this.options.hashChange!==false;this._wantsPushState=!!this.options.pushState;this._hasPushState=!!(this.options.pushState&&this.history&&this.history.pushState);var r=this.getFragment();var s=document.documentMode;var n=P.exec(navigator.userAgent.toLowerCase())&&(!s||s<=7);this.root=("/"+this.root+"/").replace(O,"/");if(n&&this._wantsHashChange){var a=e.$('