Repository: pq-dong/movierecommend Branch: master Commit: 625092676e2d Files: 76 Total size: 127.5 KB Directory structure: gitextract_xbnheaig/ ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── build.gradle ├── doc/ │ ├── ES常用命令记录.md │ ├── databaseSchema.md │ ├── 功能模块与流程.md │ ├── 大数据相关.md │ ├── 推荐算法相关.md │ ├── 数据库建表语句.md │ ├── 服务部署.md │ └── 问题汇总及解决方案.md ├── docker-compose.yml ├── gradle/ │ └── wrapper/ │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle ├── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── pqdong/ │ │ │ └── movie/ │ │ │ └── recommend/ │ │ │ ├── MovieRecommendApplication.java │ │ │ ├── annotation/ │ │ │ │ └── LoginRequired.java │ │ │ ├── config/ │ │ │ │ ├── CtrlLogAdviceAop.java │ │ │ │ ├── ElasticSearchConfig.java │ │ │ │ ├── GlobalCorsConfig.java │ │ │ │ ├── MahoutConfig.java │ │ │ │ ├── TaskConfiguration.java │ │ │ │ └── WebConfig.java │ │ │ ├── controller/ │ │ │ │ ├── CommentController.java │ │ │ │ ├── MovieController.java │ │ │ │ ├── PersonController.java │ │ │ │ ├── UserController.java │ │ │ │ └── UtilController.java │ │ │ ├── data/ │ │ │ │ ├── constant/ │ │ │ │ │ ├── ServerConstant.java │ │ │ │ │ └── UserConstant.java │ │ │ │ ├── dto/ │ │ │ │ │ ├── CommentSearchDto.java │ │ │ │ │ ├── MovieSearchDto.java │ │ │ │ │ ├── RatingDto.java │ │ │ │ │ └── UserInfo.java │ │ │ │ ├── entity/ │ │ │ │ │ ├── CommentEs.java │ │ │ │ │ ├── ConfigEntity.java │ │ │ │ │ ├── MovieEntity.java │ │ │ │ │ ├── MovieTagEntity.java │ │ │ │ │ ├── PersonEntity.java │ │ │ │ │ ├── RatingEntity.java │ │ │ │ │ └── UserEntity.java │ │ │ │ └── repository/ │ │ │ │ ├── CommentEsRepo.java │ │ │ │ ├── ConfigRepository.java │ │ │ │ ├── MovieRepository.java │ │ │ │ ├── PersonRepository.java │ │ │ │ ├── RatingRepository.java │ │ │ │ └── UserRepository.java │ │ │ ├── domain/ │ │ │ │ ├── service/ │ │ │ │ │ ├── AsyncTask.java │ │ │ │ │ └── MovieRecommender.java │ │ │ │ └── util/ │ │ │ │ └── ResponseMessage.java │ │ │ ├── exception/ │ │ │ │ ├── MyException.java │ │ │ │ └── ResultEnum.java │ │ │ ├── redis/ │ │ │ │ ├── CacheConfig.java │ │ │ │ ├── PrefixRedisSerializer.java │ │ │ │ ├── RedisApi.java │ │ │ │ └── RedisKeys.java │ │ │ ├── service/ │ │ │ │ ├── CommentService.java │ │ │ │ ├── ConfigService.java │ │ │ │ ├── ElasticSearchService.java │ │ │ │ ├── MovieService.java │ │ │ │ ├── PersonService.java │ │ │ │ ├── QiNiuService.java │ │ │ │ ├── SmsService.java │ │ │ │ └── UserService.java │ │ │ └── utils/ │ │ │ ├── LoginInterceptor.java │ │ │ ├── Md5EncryptionHelper.java │ │ │ └── RecommendUtils.java │ │ └── resources/ │ │ └── application.yml │ └── test/ │ └── java/ │ └── pqdong/ │ └── movie/ │ └── recommond/ │ ├── BaseTest.java │ ├── UtilsTest.java │ └── service/ │ └── AsyncTaskTest.java └── start.sh ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ # Compiled class file *.class # Log file *.log # BlueJ files *.ctxt # Mobile Tools for Java (J2ME) .mtj.tmp/ # Package Files # *.jar *.war *.nar *.ear *.zip *.tar.gz *.rar # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* *.DS_Store .gradle /build/ !gradle/wrapper/gradle-wrapper.jar ### STS ### .apt_generated .classpath .factorypath .project .settings .springBeans .sts4-cache ### IntelliJ IDEA ### .idea *.iws *.iml *.ipr /out/ ### NetBeans ### /nbproject/private/ /nbbuild/ /dist/ /nbdist/ /.nb-gradle/ # log /log/ /access_log/ work ================================================ FILE: Dockerfile ================================================ FROM gradle AS builder USER root ENV GRADLE_OPTS "-Dorg.gradle.daemon=false" WORKDIR / COPY . . RUN gradle bootJar FROM java:8 COPY --from=builder /build/libs/movie-recommend-* app.jar EXPOSE 10015 ENTRYPOINT ["java", "-jar", "app.jar"] ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ # movierecommend 基于Spring Boot的大数据电影推荐系统，采用协同过滤算法实现个性化推荐 ### demo地址：[http://movie.pqdong.com/#/](http://movie.pqdong.com/#/) 学生机（俗称板砖机），导致资源加载，接口响应比较慢，请耐心多等一会，让子弹多飞一会； 待功能开发完毕后会优化此页面加载速度 ### 如何在本地开发 ``` # 环境依赖 1. java环境 2. gradle项目，建议通过Intellij IDEA打开，运行build.gradle下载依赖，具体参考gradle教程 3. IDEA下载开启 lombok插件 4. 如果需要正常运行，需要使用mysql数据库和redis，具体配置可根据自己的项目配置在application.yml中 5. 发送短信和照片上传需要一些token和access_key，可以参考代码`configService.getConfigValue`获取配置和阿里云短信 ``` ### 架构 - 项目组织： 前端后端分离，通过Restful接口传递数据 - 代码组织：基于SpringBoot，采用gradle进行依赖管理 - 部署方式：采用docker部署，通过nginx实现简单的负载均衡。 - 大数据处理：采用ElasticSearch进行海量数据的全文检索 - 推荐算法： 采用Mahout基于用户的协同过滤算法和基于内容的协同过滤算法 ![项目结构图](http://ydschool-online.nos.netease.com/1582746970143Snipaste_2020-02-26_22-19-39.png) ### 技术栈 * spring boot * docker * mysql * es * redis * gradle ### 其他说明及文档 由于一直从事Golang开发，没怎么搞过java，所以决定此毕设使用java来做。其中的一些还代码有待商榷，会一点点完善。 其他文档具体可见 /doc目录 ### 数据库中数据来源声明 来源：[斗码小院公众号](http://www.csuldw.com/assets/articleImg/2019/code-main-fun.png)。 具体可见/doc/databaseSchema.md ================================================ FILE: build.gradle ================================================ buildscript { ext { springBootVersion = '2.2.0.RELEASE' } repositories { maven { url 'http://maven.aliyun.com/nexus/content/groups/public/' } maven { url 'https://plugins.gradle.org/m2/' } maven { name "elastic" url "https://artifacts.elastic.co/maven" } } dependencies { classpath("org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}") // 代码质量检测sonar classpath 'org.sonarsource.scanner.gradle:sonarqube-gradle-plugin:2.6.2' } } group = 'movie.recommend' apply plugin: 'java' apply plugin: 'idea' apply plugin: 'jacoco' apply plugin: 'application' apply plugin: 'maven-publish' apply plugin: 'org.springframework.boot' apply plugin: 'io.spring.dependency-management' apply plugin: 'org.sonarqube' jacocoTestReport { reports { xml.enabled false html.enabled true } } version = '1.8.0' ext { springCloudVersion = 'Greenwich.RELEASE' } configurations.all { // 去除boot自带的logging exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging' exclude module: 'commons-logging' } repositories { maven { url 'http://maven.aliyun.com/nexus/content/groups/public/' } maven { name "elastic" url "https://artifacts.elastic.co/maven" } mavenCentral() } dependencies { // springboot implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.springframework.boot:spring-boot-starter-actuator' implementation 'org.springframework.boot:spring-boot-starter-data-jpa' implementation 'org.springframework.boot:spring-boot-starter-data-redis' implementation 'org.springframework.boot:spring-boot-starter-log4j2' implementation 'org.springframework.boot:spring-boot-starter-aop' implementation 'org.springframework.boot:spring-boot-starter-data-elasticsearch' implementation 'org.elasticsearch.client:x-pack-transport:7.5.2' // springboot-admin implementation 'de.codecentric:spring-boot-admin-starter-client:2.1.5' annotationProcessor 'org.projectlombok:lombok:1.18.2' compileOnly 'org.projectlombok:lombok:1.18.2' testAnnotationProcessor 'org.projectlombok:lombok:1.18.2' testCompileOnly 'org.projectlombok:lombok:1.18.2' implementation 'org.apache.commons:commons-pool2:2.6.1' implementation 'com.google.guava:guava:26.0-jre' implementation 'org.apache.commons:commons-lang3:3.8' implementation 'com.alibaba:fastjson:1.2.60' implementation 'mysql:mysql-connector-java:5.1.46' implementation 'com.lmax:disruptor:3.3.6' implementation 'com.google.code.gson:gson:2.8.5' implementation 'com.aliyun:aliyun-java-sdk-core:3.5.1' implementation 'com.aliyun:aliyun-java-sdk-dysmsapi:1.1.0' implementation 'com.qiniu:qiniu-java-sdk:7.2.9' implementation 'net.sourceforge.javacsv:javacsv:2.0' //推荐相关 implementation 'org.apache.mahout:mahout-core:0.9' implementation 'org.apache.mahout:mahout-math:0.9' implementation 'org.apache.mahout:mahout-integration:0.9' // test testImplementation 'org.springframework.boot:spring-boot-starter-test' } ================================================ FILE: doc/ES常用命令记录.md ================================================ ### ES常用命令记录 ``` # 获取index下数据 /comment/_search?size=2&pretty # 查看type mapping /comment/_mapping?pretty # 统计index中数据数量 /comment/comment/_count ``` ================================================ FILE: doc/databaseSchema.md ================================================ ### 数据集简介 本数据集采集于豆瓣电影，电影与演员数据收集于2019年8月上旬，影评数据(用户、评分、评论)收集于2019年9月初，共945万数据，其中包含14万部电影，7万演员，63万用户，416万条电影评分，442万条影评！ ### 数据来源声明 爬虫项目源码： [AntSpider](https://github.com/csuldw/AntSpider)项目源码。 下载来源：[斗码小院公众号](http://www.csuldw.com/assets/articleImg/2019/code-main-fun.png)。 ### Schema ### Movie数据格式 电影数据共140502部，2019年之前的电影有139129，当前未上映的有1373部，包含21个字段，部分字段数据为空，字段说明如下: - MOVIE_ID: 电影ID - NAME: 电影名称 - ACTORS: 主演 - COVER: 封面图片地址 - DIRECTORS: 导演 - GENRES: 类型 - OFFICIAL_SITE: 地址 - REGIONS: 制片国家/地区 - LANGUAGES: 语言 - RELEASE_DATE: 上映日期 - MINS: 片长 - SCORE: 评分 - VOTES: 投票数 - TAGS: 标签 - STORYLINE: 电影描述 - YEAR: 年份 - ACTOR_IDS: 演员与PERSON_ID的对应关系,多个演员采用“\|”符号分割，格式“演员A:ID\|演员B:ID”； - DIRECTOR_IDS: 导演与PERSON_ID的对应关系,多个导演采用“\|”符号分割，格式“导演A:ID\|导演B:ID”； ## Person数据格式 Person包括演员和导演，共72959个名人数据，包含10个字段，每个PERSON_ID都会对应一个name，不存在PERSON_ID的数据已过滤，各个字段说明如下: - PERSON_ID: 名人ID - NAME: 演员名称 - SEX: 性别 - NAME_EN: 更多英文名 - NAME_ZH: 更多中文名 - BIRTH: 出生日期 - BIRTHPLACE: 出生地 - CONSTELLATORY: 星座 - PROFESSION: 职业 - BIOGRAPHY: 简介，存在简介数据的名人只有15135个。 ## User数据格式 639125用户数据，包含4个字段，具体的字段如下： - USER_ID：用户ID - USER_MD: 用户md5，唯一标示，这里设计的及其不好，但是数据集以此为标示，因此我也采用 - USER_NICKNAME: 用户昵称 - USER_AVATAR: 用户头像 - USER_Tags: 用户标签 ### 标签表 - TAG_ID: 标签id - TAG_NAME: 标签名称 ### Rating数据 600384个用户的4169420条评分数据，涉及电影68471部，评分值为1-5分（1-很差，2-较差，3-还行，4-推荐，5-力荐），共包含5个字段，数据格式如下： - RATING_ID: 评分ID - USER_ID：豆瓣用户ID - MOVIE_ID: 电影ID，对应豆瓣的DOUBAN_ID - RATING: 评分 - RATING_TIME: 评分时间 ### Comment数据格式 评论数据共4428475 条，包含6个字段，各字段说明: - COMMENT_ID: 评论ID - USER_MD：用户ID - USERNAME: 用户名称 - USERAVATAR： 用户头像 - MOVIE_ID: 电影ID - MOVIE_NAME: 电影名称 - CONTENT: 评论内容 - VOTES: 评论赞同数 - COMMENT_TIME: 评论时间 ================================================ FILE: doc/功能模块与流程.md ================================================ ## 功能模块与流程 ### 首页 1. 首页主要展示热门电影，最新，高分，好评等多种标签电影。通过ES mapping进行查询 2. Es mapping组合查询要优于mysql的组合查询 3. 首页热门推荐：数据产生流程具体下 4. 任务点： - [ ] 电影详情页接口 - [x] 演员详情页接口 ### 热评 1. 目前从数据库通过movie_id直接load 2. 任务点 - [ ] 评论详情页 - [ ] 电影排名详情页 ### 用户 - [x] userLogin接口 - [x] userLogout接口 - [x] userInfo - [ ] userEdit ### 数据流转流程 1. 用户对自身打标签 2. 用户搜索或者过滤产生行为信息 3. 用户对电影打分，评价：获取电影信息 4. 将以上信息通过kafka->Flink聚合，进行推荐算法处理->es ### 难点 ES集群搭建需要比较好的服务器 ================================================ FILE: doc/大数据相关.md ================================================ ### 大数据相关 ### 1. **java8 stream流式API的使用**，加快了处理常见集合的速度，使得在处理数据时更快，更方便简洁 ```java public Map searchMovies(MovieSearchDto info) { Pair pair = RecommendUtils.getStartAndEnd(info.getPage(), info.getSize()); List allMovie = info.getTags().stream() .map(t -> getMovies(t, "tag", info.getPage() * info.getSize())) .flatMap(Collection::stream) .collect(Collectors.toCollection(LinkedList::new)); if (!StringUtils.isEmpty(info.getContent())) { allMovie.addAll(getMovies(info.getContent(), "name", info.getPage() * info.getSize())); } List movieList = allMovie.subList(pair.getLeft(), pair.getRight() <= allMovie.size() ? pair.getRight() : allMovie.size()); Map result = new HashMap<>(2, 1); result.put("total", allMovie.size()); result.put("movieList", movieList.stream().peek(m -> { if (StringUtils.isEmpty(m.getCover())) { m.setCover(ServerConstant.DefaultImg); } }).collect(Collectors.toCollection(LinkedList::new))); return result; } ``` ### 2. 评论和观看记录是大数据，因此采用es处理，部署启动es ``` docker run -d \ --name es \ -p 9200:9200 -p 9300:9300 \ -e "discovery.type=single-node" -e ES_JAVA_OPTS="-Xms200m -Xmx200m" \ -e "xpack.security.enabled=true" \ -v /home/movie/esdata:/usr/share/elasticsearch/data \ elasticsearch:7.5.2 ``` 以上方式会报错，查看日志发现是权限问题，因为我们在打数据卷时权限和容器里es，需要的权限是对不上的，所以把容器外数据卷的权限扩大一下， 保证容器内对外可写就行。 最好设置密码： ``` # 进入容器 docker exec -it es /bin/sh cd bin/ elasticsearch-setup-passwords interactive # 设置成功后 Changed password for user [apm_system] Changed password for user [kibana] Changed password for user [logstash_system] Changed password for user [beats_system] Changed password for user [remote_monitoring_user] Changed password for user [elastic] ``` ### 3. 关于使用es踩了一个大坑 如上使用了最新版的es，并且使用了xpack来进行权限验证，保证数据安全。但是spring boot 2.2.x版本之后才开始支持 es 7.x版本即以上， 一. 因此首先将spring boot的版本升级到 2.2.0 二. `org.springframework.boot:spring-boot-starter-data-elasticsearch`使用默认的TransportClient是不进行权限校验的，因此链接es失败。 关于处理方法，网上一大堆，大部分都是复制粘贴，甚至全是错误的。经过自己摸索，解决了此问题，在此做一下汇总，日后处理。 ``` 1. 首先在maven或者gradle中需要引入xpack依赖包，需要注意在repositories添加https://artifacts.elastic.co/maven源，具体可见build.gradle 网上说需要去除transport依赖，但是我在用时去除依赖会导致某些类加载不到。因此不用去除。 implementation 'org.springframework.boot:spring-boot-starter-data-elasticsearch' implementation 'org.elasticsearch.client:x-pack-transport:7.5.2' 2.application中添加es配置 spring: data: elasticsearch: repositories: enabled: true elasticsearch: ip: es结点ip，这个是用来注入配置的，具体见ElasticSearchConfig rest: uris: ip:port # port结点 username: yourname # es用户名 password: yourpassword # es密码 3. 配置bean，具体见ElasticSearchConfig ``` 三. 创建CommentEsRepo，否则是不会自动创建index的，同时还要注意在CommentEs中添加es字段的type 四. 简单的查询可以直接通过CommentEsRepo来实现，复杂的查询需要构造query然后通过ElasticsearchRestTemplate查询即可。 ElasticsearchRestTemplate是ElasticsearchOperations 接口的另一个实现，其是基于High Level REST Client实现的，所有具有更高的性能 ### Comment评论ES index选型 1. comment中主要包含评论内容，电影id，用户头像，昵称等信息，其中用户头像和昵称是可变的，也就是用户在修改昵称和头像后，在返回给前端时应该是变化后的值 2. 因此在对comment index选型时有一下几种方案： 方案 | 对比 ---|--- Application-side joins | 应用层连接，独立了文档，但对应用层负载压力增大 Data denormalization | 非规范化数据，通过冗余来扁平化文档 Nested objects | 嵌套对象，文档不可以单独存在，是隐含在父文档中，需要单独建立文档 Parent/child relationships | 父子文档，文档独立，但是需要在同一个分片中，对内存要求高 3. 考虑到服务器性能，以及业务场景，决定采用Data denormalization的方式，即将用户信息扁平处理到Comment index中，但是这样会带来另一个较为棘手的问题 当更新用户信息时需要对comment文档中的相关记录做更新，目前可以通过logstash来同步， 但是考虑到配置复杂，且用户信息更新是一个较不频繁的操作，暂时不采用这种方式。 所以直接在应用层通过异步线程池批量更新的方式去维护 ================================================ FILE: doc/推荐算法相关.md ================================================ ### 推荐算法相关 ### Apache Mahout Apache Mahout 是 Apache Software Foundation（ASF） 旗下的一个开源项目，提供一些可扩展的机器学习领域经典算法的实现， 旨在帮助开发人员更加方便快捷地创建智能应用程序...具体参考官网[https://www.ibm.com/developerworks/cn/java/j-lo-mahout/](https://www.ibm.com/developerworks/cn/java/j-lo-mahout/) ### 处理方法 用户对每一个电影的评分，可以等同于用户对此电影的喜好程度； 对电影评分表中的数据采用协同过滤算法生成一个推荐列表，存储在redis中 目前考虑到服务器性能，没有做成实时推荐，推荐结果放到了redis中做一天的缓存 具体代码可见 /src/java/main/pqdong/movie/recommend/domain/service ### 基于用户的协同过滤算法 核心代码： ```java public List userBasedRecommender(long userID,int size) throws TasteException { UserSimilarity similarity = new EuclideanDistanceSimilarity(dataModel ); NearestNUserNeighborhood neighbor = new NearestNUserNeighborhood(NEIGHBORHOOD_NUM, similarity, dataModel ); Recommender recommender = new CachingRecommender(new GenericUserBasedRecommender(dataModel , neighbor, similarity)); List recommendations = recommender.recommend(userID, size); return getRecommendedItemIDs(recommendations); } ``` ### 基于内容的协同过滤算法 ```java public List itemBasedRecommender(long userID,int size) throws TasteException { List recommendItems = new ArrayList<>(); ItemSimilarity itemSimilarity = new PearsonCorrelationSimilarity(dataModel); Recommender recommender = new GenericItemBasedRecommender(dataModel, itemSimilarity); List recommendations = recommender.recommend(userID, size); return getRecommendedItemIDs(recommendations); } ``` ================================================ FILE: doc/数据库建表语句.md ================================================ ### 数据库建表语句 数据来源：参考/doc/databaseSchema.md文档 数据库建议使用Mysql 5.6+。 项目中使用`@Table`注解，当数据库正确配置并连接时，spring boot会自动创建相关的数据库表， 具体可以参考 `/data/entity`和`application.yml` 1. user表建表语句 ```sql CREATE TABLE `user` ( `user_id` bigint(20) NOT NULL AUTO_INCREMENT, `password` varchar(255) DEFAULT NULL, `user_avatar` varchar(255) DEFAULT NULL, `user_md` varchar(50) DEFAULT NULL, `user_nickname` varchar(50) DEFAULT NULL, `user_tags` varchar(255) DEFAULT NULL, `phone` varchar(255) DEFAULT NULL, `motto` varchar(255) DEFAULT NULL, `sex` varchar(255) DEFAULT NULL, PRIMARY KEY (`user_id`), UNIQUE KEY `user_md` (`user_md`,`user_nickname`) ) ENGINE=InnoDB AUTO_INCREMENT=195086 DEFAULT CHARSET=utf8mb4 ``` 2. movie表 ```sql CREATE TABLE `movie` ( `movie_id` bigint(20) NOT NULL AUTO_INCREMENT, `actor_ids` varchar(255) DEFAULT NULL, `actors` varchar(255) DEFAULT NULL, `cover` varchar(255) DEFAULT NULL, `director_ids` varchar(255) DEFAULT NULL, `directors` varchar(255) DEFAULT NULL, `genres` varchar(255) DEFAULT NULL, `languages` varchar(255) DEFAULT NULL, `mins` int(11) DEFAULT NULL, `name` varchar(255) DEFAULT NULL, `official_site` varchar(255) DEFAULT NULL, `regions` varchar(255) DEFAULT NULL, `release_date` date DEFAULT NULL, `score` float DEFAULT NULL, `storyline` text, `tags` varchar(255) DEFAULT NULL, `votes` int(11) DEFAULT NULL, `year` int(11) DEFAULT NULL, PRIMARY KEY (`movie_id`), KEY `actor` (`actor_ids`(191)), KEY `index1` (`score`), KEY `index2` (`tags`(191)) ) ENGINE=InnoDB AUTO_INCREMENT=34782321 DEFAULT CHARSET=utf8mb4 ``` 3. person表 ```sql CREATE TABLE `person` ( `biography` varchar(255) DEFAULT NULL, `birth` varchar(255) DEFAULT NULL, `constellation` varchar(255) DEFAULT NULL, `name` varchar(255) DEFAULT NULL, `profession` varchar(255) DEFAULT NULL, `sex` varchar(255) DEFAULT NULL, `birth_place` varchar(255) DEFAULT NULL, `name_en` varchar(255) DEFAULT NULL, `name_zn` varchar(255) DEFAULT NULL, `person_id` bigint(20) NOT NULL AUTO_INCREMENT, `avatar` varchar(255) DEFAULT NULL, PRIMARY KEY (`person_id`), KEY `index1` (`name`(191)) ) ENGINE=InnoDB AUTO_INCREMENT=1422443 DEFAULT CHARSET=utf8mb4 ``` 4. movie_tags ```sql CREATE TABLE `movie_tags` ( `movie_tag_id` bigint(20) NOT NULL AUTO_INCREMENT, `name` varchar(255) DEFAULT NULL, `value` varchar(255) DEFAULT NULL, PRIMARY KEY (`movie_tag_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ``` 5. rating ```sql CREATE TABLE `rating` ( `rating_id` bigint(20) NOT NULL AUTO_INCREMENT, `movie_id` bigint(20) DEFAULT NULL, `rating` int(11) DEFAULT NULL, `time` date DEFAULT NULL, `user_id` bigint(20) DEFAULT NULL, PRIMARY KEY (`rating_id`) ) ENGINE=InnoDB AUTO_INCREMENT=1970708 DEFAULT CHARSET=utf8mb4 ``` 6. config ```sql CREATE TABLE `config` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `key` varchar(255) DEFAULT NULL, `value` varchar(255) DEFAULT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8mb4 ``` ================================================ FILE: doc/服务部署.md ================================================ ### 服务部署 1. 服务部署与管理 V1: 目前分别为前后端起了两个容器， 因为需要暴露端口，以及方便联调，所以docker-compose方式暂时不启用 为了解决flink，kafka，es部署对服务器性能的要求，决定将其部署到另外一台服务器 V2: 暴露redis端口到外网不安全，因此还是使用docker-compose 2. 部署命令 ``` # 后端，采用host模式 docker run -d --name=movierecommend -p 10015:10015 --network=host -v /home/movie/log:/access_log imagename # 前端，采用host模式 docker run -d --name=moviefront -p 10016:10016 --network=host imagename # redis容器，采用docker bridge模式，同时iptables禁掉远程访问6379端口，开启aof保存数据 docker run -d --name=redis -p 127.0.0.1:6379:6379 -v /home/movie/redisdata:/data redis redis-server --appendonlyyes ``` 3. nginx转发 配置了二级域名，并通过nginx转发。 ================================================ FILE: doc/问题汇总及解决方案.md ================================================ ## 问题汇总及解决方案 ### 1. user表性能分析与数据处理 1. user表的主键为user_id,但是实际业务中标示用户的唯一字段为user_md（md5序列），因此需要在user_md上创建一个普通索引来加快查询速度。 在导入数据的时候，发现有一部分数据nickname重复，在用nickname和password做登录时，因为nickname不唯一，因此会造成登录错误，所以需要对数据进行清洗， 清洗方式如下： ```sql UPDATE `user` set user_nickname=CONCAT(user_nickname,user_id) WHERE user_nickname in (SELECT u3.user_nickname FROM (SELECT u2.user_nickname from `user` u2 GROUP BY u2.user_nickname HAVING COUNT(*)>1) AS u3) ``` 对nickname的唯一性维护在程序中实现，不在数据层面做约束，从而降低数据库创建索引的存储和时间消耗。 ### 登录问题 用户登录后会以 key: token value:userMd的信息向redis中保存；登录后前端在request的header中带着token, 后端通过自定义注解@LoginRequrie来进行登录验证（根据token获取userMd然后查询用户是否存在） ### 部署问题 服务器宽带太小，直接在服务器上使用docker build构建镜像速度太慢，所以搞了个github的公开仓库，来存储和分发镜像。 并使用开发机构建镜像，push到github上来加速构建和部署速度。 同时修改了服务器的docker代理，来加速镜像的拉取。 ### 性能问题 1、发现在更新用户信息时接口响应较慢，排查发现user表没有创建索引(肯定是智障了....)，因此对user表user_md和user_nickname创建唯一索引 ### 其他问题 发现服务器内存使用过高，导致docker服务oom，查看当前进程的内存使用情况发现node服务占用的资源太多，先kill掉。 然后将问题定位到前端服务部署方式，在原来的方式中启动了一个node服务，然后使用 `npm run dev`直接启动了前端项目，这样导致占用资源太多。 解决方法，直接使用`npm run build`将静态资源打出来，起个http的服务，暴露资源就可以 同时启用swap，设置大小为2G （ps: 因为服务器太弱鸡踩了好多坑，好像有台大的服务器，可惜我没有钱...太惨了...） ================================================ FILE: docker-compose.yml ================================================ version: "3" services: redis: image: redis container_name: redis:4.0-alpine restart: always volumes: - /home/movie/redis:/data backend: image: backend:1.0 container_name: movierecommend depends_on: - redis volumes: - /home/movie/log:/access_log network_mode: host front: image: moviefront:1.0 container_name: moviefront depends_on: - backend ports: - "10016:10016" network_mode: host ================================================ FILE: gradle/wrapper/gradle-wrapper.properties ================================================ #Thu Feb 27 02:35:42 CST 2020 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-all.zip ================================================ FILE: gradlew ================================================ #!/usr/bin/env sh ############################################################################## ## ## Gradle start up script for UN*X ## ############################################################################## # Attempt to set APP_HOME # Resolve links: $0 may be a link PRG="$0" # Need this for relative symlinks. while [ -h "$PRG" ] ; do ls=`ls -ld "$PRG"` link=`expr "$ls" : '.*-> \(.*\)$'` if expr "$link" : '/.*' > /dev/null; then PRG="$link" else PRG=`dirname "$PRG"`"/$link" fi done SAVED="`pwd`" cd "`dirname \"$PRG\"`/" >/dev/null APP_HOME="`pwd -P`" cd "$SAVED" >/dev/null APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" warn () { echo "$*" } die () { echo echo "$*" echo exit 1 } # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false case "`uname`" in CYGWIN* ) cygwin=true ;; Darwin* ) darwin=true ;; MINGW* ) msys=true ;; NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables JAVACMD="$JAVA_HOME/jre/sh/java" else JAVACMD="$JAVA_HOME/bin/java" fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else JAVACMD="java" which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi # Increase the maximum file descriptors if we can. if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then MAX_FD_LIMIT=`ulimit -H -n` if [ $? -eq 0 ] ; then if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then MAX_FD="$MAX_FD_LIMIT" fi ulimit -n $MAX_FD if [ $? -ne 0 ] ; then warn "Could not set maximum file descriptor limit: $MAX_FD" fi else warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" fi fi # For Darwin, add options to specify how the application appears in the dock if $darwin; then GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" fi # For Cygwin, switch paths to Windows format before running java if $cygwin ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` JAVACMD=`cygpath --unix "$JAVACMD"` # We build the pattern for arguments to be converted via cygpath ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` SEP="" for dir in $ROOTDIRSRAW ; do ROOTDIRS="$ROOTDIRS$SEP$dir" SEP="|" done OURCYGPATTERN="(^($ROOTDIRS))" # Add a user-defined pattern to the cygpath arguments if [ "$GRADLE_CYGPATTERN" != "" ] ; then OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" fi # Now convert the arguments - kludge to limit ourselves to /bin/sh i=0 for arg in "$@" ; do CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` else eval `echo args$i`="\"$arg\"" fi i=$((i+1)) done case $i in (0) set -- ;; (1) set -- "$args0" ;; (2) set -- "$args0" "$args1" ;; (3) set -- "$args0" "$args1" "$args2" ;; (4) set -- "$args0" "$args1" "$args2" "$args3" ;; (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; esac fi # Escape application args save () { for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done echo " " } APP_ARGS=$(save "$@") # Collect all arguments for the java command, following the shell quoting and substitution rules eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then cd "$(dirname "$0")" fi exec "$JAVACMD" "$@" ================================================ FILE: gradlew.bat ================================================ @if "%DEBUG%" == "" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @rem @rem ########################################################################## @rem Set local scope for the variables with windows NT shell if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 if "%DIRNAME%" == "" set DIRNAME=. set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. set DEFAULT_JVM_OPTS="-Xmx64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if "%ERRORLEVEL%" == "0" goto init echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. echo. echo Please set the JAVA_HOME variable in your environment to match the echo location of your Java installation. goto fail :findJavaFromJavaHome set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto init echo. echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% echo. echo Please set the JAVA_HOME variable in your environment to match the echo location of your Java installation. goto fail :init @rem Get command-line arguments, handling Windows variants if not "%OS%" == "Windows_NT" goto win9xME_args :win9xME_args @rem Slurp the command line arguments. set CMD_LINE_ARGS= set _SKIP=2 :win9xME_args_slurp if "x%~1" == "x" goto execute set CMD_LINE_ARGS=%* :execute @rem Setup the command line set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar @rem Execute Gradle "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% :end @rem End local scope for the variables with windows NT shell if "%ERRORLEVEL%"=="0" goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 exit /b 1 :mainEnd if "%OS%"=="Windows_NT" endlocal :omega ================================================ FILE: settings.gradle ================================================ rootProject.name = 'movie-recommend' ================================================ FILE: src/main/java/pqdong/movie/recommend/MovieRecommendApplication.java ================================================ package pqdong.movie.recommend; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.scheduling.annotation.EnableAsync; @EnableAsync @SpringBootApplication() public class MovieRecommendApplication { public static void main(String[] args) { SpringApplication.run(MovieRecommendApplication.class, args); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/annotation/LoginRequired.java ================================================ package pqdong.movie.recommend.annotation; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; /** * @author pqdong * @description * @date 2020/03/04 */ @Target({ElementType.METHOD}) @Retention(RetentionPolicy.RUNTIME) public @interface LoginRequired { } ================================================ FILE: src/main/java/pqdong/movie/recommend/config/CtrlLogAdviceAop.java ================================================ package pqdong.movie.recommend.config; import lombok.extern.slf4j.Slf4j; import org.aspectj.lang.ProceedingJoinPoint; import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; import org.aspectj.lang.reflect.MethodSignature; import org.springframework.stereotype.Component; import org.springframework.web.bind.annotation.*; import java.util.Arrays; @Slf4j @Aspect @Component public class CtrlLogAdviceAop { @Around("@annotation(requestMapping)") public Object requestMappingAdvice(ProceedingJoinPoint thisJoinPoint, RequestMapping requestMapping) throws Throwable { String path = requestMapping.value().length == 0 ? "" : requestMapping.value()[0]; return process(thisJoinPoint, path); } @Around("@annotation(getMapping)") public Object getMappingAdvice(ProceedingJoinPoint thisJoinPoint, GetMapping getMapping) throws Throwable { String path = getMapping.value().length == 0 ? "" : getMapping.value()[0]; return process(thisJoinPoint, path); } @Around("@annotation(postMapping)") public Object postMappingAdvice(ProceedingJoinPoint thisJoinPoint, PostMapping postMapping) throws Throwable { String path = postMapping.value().length == 0 ? "" : postMapping.value()[0]; return process(thisJoinPoint, path); } @Around("@annotation(putMapping)") public Object putMappingAdvice(ProceedingJoinPoint thisJoinPoint, PutMapping putMapping) throws Throwable { String path = putMapping.value().length == 0 ? "" : putMapping.value()[0]; return process(thisJoinPoint, path); } @Around("@annotation(deleteMapping)") public Object deleteMappingAdvice(ProceedingJoinPoint thisJoinPoint, DeleteMapping deleteMapping) throws Throwable { String path = deleteMapping.value().length == 0 ? "" : deleteMapping.value()[0]; return process(thisJoinPoint, path); } private Object process(ProceedingJoinPoint pjp, String path) throws Throwable { long startTime = System.currentTimeMillis(); Object result = pjp.proceed(); long endTime = System.currentTimeMillis(); MethodSignature methodSignature = (MethodSignature) pjp.getSignature(); String logName = methodSignature.getMethod().getName() + "(" + path + ")"; log.info(logName + ".time=" + (endTime - startTime)); log.info(logName + " Args: " + Arrays.toString(pjp.getArgs())); return result; } } ================================================ FILE: src/main/java/pqdong/movie/recommend/config/ElasticSearchConfig.java ================================================ package pqdong.movie.recommend.config; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import pqdong.movie.recommend.service.ConfigService; import javax.annotation.Resource; import java.net.InetSocketAddress; public class ElasticSearchConfig { @Resource private ConfigService configService; @Bean public TransportClient transportClient(@Value("${spring.elasticsearch.ip}") String ip){ String password = configService.getConfigValue("ESPASSWORD"); try (TransportClient client = new PreBuiltXPackTransportClient(Settings.builder() .put("cluster.name", "docker-cluster") .put("xpack.security.user", password) .put("timeout", 10000) .put("client.transport.ping_timeout", 10000) .build()) .addTransportAddress(new TransportAddress(new InetSocketAddress(ip, 9300)))) { return client; } } } ================================================ FILE: src/main/java/pqdong/movie/recommend/config/GlobalCorsConfig.java ================================================ package pqdong.movie.recommend.config; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.web.cors.CorsConfiguration; import org.springframework.web.cors.UrlBasedCorsConfigurationSource; import org.springframework.web.filter.CorsFilter; @Configuration public class GlobalCorsConfig { @Bean public CorsFilter corsFilter() { CorsConfiguration config = new CorsConfiguration(); config.addAllowedOrigin("*"); config.setAllowCredentials(true); config.addAllowedMethod("*"); config.addAllowedHeader("*"); // config.addExposedHeader("*"); UrlBasedCorsConfigurationSource configSource = new UrlBasedCorsConfigurationSource(); configSource.registerCorsConfiguration("/**", config); return new CorsFilter(configSource); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/config/MahoutConfig.java ================================================ package pqdong.movie.recommend.config; import com.mysql.jdbc.jdbc2.optional.MysqlDataSource; import org.apache.mahout.cf.taste.impl.model.file.FileDataModel; import org.apache.mahout.cf.taste.impl.model.jdbc.MySQLJDBCDataModel; import org.apache.mahout.cf.taste.model.DataModel; import org.springframework.beans.factory.annotation.Autowire; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import javax.sql.DataSource; @Configuration public class MahoutConfig { @Autowired private DataSource dataSource; @Bean(autowire = Autowire.BY_NAME,value = "mySQLDataModel") public DataModel getMySQLJDBCDataModel(){ return new MySQLJDBCDataModel(dataSource,"rating","user_id", "movie_id","rating", "time"); } // @Bean(autowire = Autowire.BY_NAME,value = "fileDataModel") // public DataModel getDataModel() throws IOException { // URL url=MahoutConfig.class.getClassLoader().getResource("/rating.csv"); // return new FileDataModel(new File(Objects.requireNonNull(url).getFile())); // } } ================================================ FILE: src/main/java/pqdong/movie/recommend/config/TaskConfiguration.java ================================================ package pqdong.movie.recommend.config; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; /** * TaskConfiguration * * @author pqdong * @description 异步线程池配置 * @since 2020/03/31 */ @Configuration public class TaskConfiguration { // 设置下线程池大小，防止oom @Bean("taskExecutor") public Executor taskExecutor() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(5); executor.setMaxPoolSize(20); executor.setQueueCapacity(50); executor.setKeepAliveSeconds(60); executor.setThreadNamePrefix("taskExecutor-"); executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); return executor; } } ================================================ FILE: src/main/java/pqdong/movie/recommend/config/WebConfig.java ================================================ package pqdong.movie.recommend.config; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.web.servlet.config.annotation.InterceptorRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; import pqdong.movie.recommend.utils.LoginInterceptor; /** * WebConfig * * @author pqdong * @since 2020/03/09 */ @Configuration public class WebConfig implements WebMvcConfigurer { public LoginInterceptor loginInterceptor() { return new LoginInterceptor(); } @Override public void addInterceptors(InterceptorRegistry registry) { registry.addInterceptor(loginInterceptor()); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/controller/CommentController.java ================================================ package pqdong.movie.recommend.controller; import org.springframework.web.bind.annotation.*; import pqdong.movie.recommend.annotation.LoginRequired; import pqdong.movie.recommend.data.dto.CommentSearchDto; import pqdong.movie.recommend.data.entity.CommentEs; import pqdong.movie.recommend.domain.util.ResponseMessage; import pqdong.movie.recommend.service.CommentService; import javax.annotation.Resource; @RestController @RequestMapping("/comment") public class CommentController { @Resource private CommentService commentService; /** * @method getCommentList 获取电影标签 */ @PostMapping("/list") public ResponseMessage getCommentList(@RequestBody CommentSearchDto commentSearchDto) { return ResponseMessage.successMessage(commentService.getCommentList(commentSearchDto)); } /** * @method submitComment 提交评论 */ @PostMapping("/submit") @LoginRequired public ResponseMessage submitComment(@RequestBody CommentEs commentEs) { CommentEs comment = commentService.submitComment(commentEs); if (comment == null){ return ResponseMessage.failedMessage("留言过快或失败，请稍后重试！"); } return ResponseMessage.successMessage(comment); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/controller/MovieController.java ================================================ package pqdong.movie.recommend.controller; import org.apache.commons.lang3.StringUtils; import org.springframework.web.bind.annotation.*; import pqdong.movie.recommend.annotation.LoginRequired; import pqdong.movie.recommend.data.dto.MovieSearchDto; import pqdong.movie.recommend.data.dto.RatingDto; import pqdong.movie.recommend.data.entity.UserEntity; import pqdong.movie.recommend.domain.util.ResponseMessage; import pqdong.movie.recommend.service.MovieService; import javax.annotation.Resource; @RestController @RequestMapping("/movie") public class MovieController { @Resource private MovieService movieService; /** * @method getMovieTags 获取电影标签 */ @GetMapping("/tag") public ResponseMessage get() { return ResponseMessage.successMessage(movieService.getMovieTags()); } /** * @method allMovie 获取电影列表 * @param key 关键字 * @param page 当前页数 * @param size 每页数据量 **/ @GetMapping("/list") public ResponseMessage allMovie( @RequestParam(required = false, defaultValue = "") String key, @RequestParam(required = false, defaultValue = "1") int page, @RequestParam(required = false, defaultValue = "12") int size) { return ResponseMessage.successMessage(movieService.getAllMovie(key, page, size)); } /** * @method getMovie 获取电影详情 * @param movieId 电影id **/ @GetMapping("/info") public ResponseMessage getMovie( @RequestParam(required = true, defaultValue = "0") Long movieId) { return ResponseMessage.successMessage(movieService.getMovie(movieId)); } /** * @param personName 演员id * @method getPersonMovie 获取演员出演的电影 **/ @GetMapping("/person/attend") public ResponseMessage getPersonAttendMovie( @RequestParam(required = true, defaultValue = "0") String personName) { return ResponseMessage.successMessage(movieService.getPersonAttendMovie(personName)); } /** * @param info 查找条件 * @method getMovieListByTag 根据标签获取电影列表 **/ @PostMapping("/listByTag") public ResponseMessage getMovieListByTag(@RequestBody(required = true) MovieSearchDto info) { if (info.getTags().isEmpty() && StringUtils.isEmpty(info.getContent())){ return ResponseMessage.successMessage(movieService.getAllMovie("", info.getPage(), info.getSize())); }else{ return ResponseMessage.successMessage(movieService.searchMovies(info)); } } /** * @method getHighMovie 获取高分电影 **/ @GetMapping("/high") public ResponseMessage getHighMovie() { return ResponseMessage.successMessage(movieService.getHighMovie()); } /** * @param rating 打分 * @method updateScore 对电影评分 **/ @PostMapping("/update") @LoginRequired public ResponseMessage updateScore(@RequestBody(required = true) RatingDto rating) { return ResponseMessage.successMessage(movieService.updateScore(rating)); } /** * @method getHighMovie 获取高分电影 **/ @PostMapping("/recommend") public ResponseMessage getRecommendMovie(@RequestBody(required = false) UserEntity user) { return ResponseMessage.successMessage(movieService.getRecommendMovie(user)); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/controller/PersonController.java ================================================ package pqdong.movie.recommend.controller; import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import pqdong.movie.recommend.domain.util.ResponseMessage; import pqdong.movie.recommend.service.PersonService; import javax.annotation.Resource; @RestController @Slf4j @RequestMapping("/person") public class PersonController { @Resource private PersonService personService; /** * @param key 关键字 * @param page 当前页数 * @param size 每页数据量 * @method allPerson 查看所有演员 **/ @GetMapping("/list") public ResponseMessage allPerson( @RequestParam(required = false, defaultValue = "") String key, @RequestParam(required = false, defaultValue = "1") int page, @RequestParam(required = false, defaultValue = "4") int size) { return ResponseMessage.successMessage(personService.getAllPerson(key, page, size)); } /** * @param personId 演员id * @method getPerson 获取导演演员详情 **/ @GetMapping("/info") public ResponseMessage getPerson( @RequestParam(required = true, defaultValue = "0") Long personId) { return ResponseMessage.successMessage(personService.getPerson(personId)); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/controller/UserController.java ================================================ package pqdong.movie.recommend.controller; import org.apache.commons.lang3.StringUtils; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; import pqdong.movie.recommend.annotation.LoginRequired; import pqdong.movie.recommend.data.entity.UserEntity; import pqdong.movie.recommend.data.dto.UserInfo; import pqdong.movie.recommend.domain.util.ResponseMessage; import pqdong.movie.recommend.service.SmsService; import pqdong.movie.recommend.service.UserService; import javax.annotation.Resource; import java.util.Map; /** * UserController * @description 用户信息相关接口 * @author pqdong * @since 2020/02/27 16:42 */ @RestController @RequestMapping("/user") public class UserController { @Resource private SmsService smsService; @Resource private UserService userService; /** * @method getUserInfo 获取用户信息 */ @GetMapping("/userInfo") public ResponseMessage getCourseInfo(@RequestParam(required = true) String token) { return ResponseMessage.successMessage(userService.getUserInfo(token)); } /** * @method updateUserInfo 修改用户信息 */ @PostMapping("/userInfo") public ResponseMessage updateUserInfo(@RequestBody(required = true) UserEntity user) { UserEntity userInfo = userService.updateUser(user); if (null == userInfo){ return ResponseMessage.failedMessage("昵称已经存在，请跟换昵称！"); } return ResponseMessage.successMessage(userService.updateUser(user)); } /** * @method register 注册用户 */ @PostMapping("/register") public ResponseMessage register(@RequestBody UserInfo user){ String result = userService.register(user); if (result.equals("success")){ return ResponseMessage.successMessage("success"); } else{ return ResponseMessage.failedMessage(result); } } /** * @method login 登录接口 */ @PostMapping("/login") public ResponseMessage userLogin(@RequestBody UserInfo user) { Map info = userService.login(user.getUsername(), user.getPassword()); if (info != null){ return ResponseMessage.successMessage(info); } else { return ResponseMessage.failedMessage("登录失败，请检查用户名或密码！"); } } /** * @method code 发送短信验证码的接口 * @param phone 手机号 **/ @GetMapping("/code") public ResponseMessage code(@RequestParam String phone) { String code = smsService.sendCode(phone); if (StringUtils.isNotEmpty(code)) { return ResponseMessage.successMessage("发送成功"); } else { return ResponseMessage.failedMessage("发送失败"); } } /** * @method upload 上传用户头像 * @param avatar 头像 **/ @PostMapping("/avatar") @LoginRequired public ResponseMessage upload(@RequestParam("userMd") String userMd, @RequestParam("avatar") MultipartFile avatar) { String url = userService.uploadAvatar(userMd, avatar); if (StringUtils.contains(url,"http")) { return ResponseMessage.successMessage(url); } else { return ResponseMessage.failedMessage(url); } } /** * @method logout 退出接口 **/ @PostMapping("/logout") @LoginRequired public ResponseMessage logout() { return ResponseMessage.successMessage(userService.logout()); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/controller/UtilController.java ================================================ package pqdong.movie.recommend.controller; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import pqdong.movie.recommend.domain.util.ResponseMessage; import pqdong.movie.recommend.service.ElasticSearchService; import javax.annotation.Resource; /** * UtilController * @description 系统状态请求 * @author pqdong * @since 2020/02/27 16:42 */ @RestController @RequestMapping("/util") public class UtilController { @Resource private ElasticSearchService elasticSearchService; @GetMapping("/ping/system") public ResponseMessage pingSystem() { return ResponseMessage.successMessage("system health"); } @GetMapping("/ping/es") public ResponseMessage pingEs() { return ResponseMessage.successMessage(elasticSearchService.getAllIndex()); } @GetMapping("/backend/comment") public ResponseMessage backend() { return ResponseMessage.successMessage(elasticSearchService.importCommentToEs()); } @GetMapping("/update/comment") public ResponseMessage update(@RequestParam(required = false) Long movieId) { return ResponseMessage.successMessage(elasticSearchService.updateCommentToEs(movieId)); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/constant/ServerConstant.java ================================================ package pqdong.movie.recommend.data.constant; /* * 系统配置 * @author pqdong * @time 2020/03/28 */ public class ServerConstant { public static final String DefaultImg = "https://ydschool-video.nosdn.127.net/1585389729635Snipaste_2020-03-28_18-02-41.png"; } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/constant/UserConstant.java ================================================ package pqdong.movie.recommend.data.constant; import com.google.common.base.Joiner; public class UserConstant { public static final String OK = "OK"; public static final String LOGOUT = "logout"; public static final String USER_AVATAR = "avatar"; public static final String PHONE_CODE = "phone_code"; } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/dto/CommentSearchDto.java ================================================ package pqdong.movie.recommend.data.dto; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import java.util.List; @Data @AllArgsConstructor @NoArgsConstructor public class CommentSearchDto { private Integer page; private Integer size; private Long movieId; private String userMd; private String content; } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/dto/MovieSearchDto.java ================================================ package pqdong.movie.recommend.data.dto; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import java.util.List; @Data @AllArgsConstructor @NoArgsConstructor public class MovieSearchDto { private Integer page; private Integer size; private List tags; private String content; } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/dto/RatingDto.java ================================================ package pqdong.movie.recommend.data.dto; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import java.util.List; @Data @AllArgsConstructor @NoArgsConstructor public class RatingDto { private Long movieId; private Long userId; private Float rating; } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/dto/UserInfo.java ================================================ package pqdong.movie.recommend.data.dto; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import pqdong.movie.recommend.data.entity.MovieTagEntity; import java.util.List; @Data // 自动生成无参数构造函数 @NoArgsConstructor // 自动生成全参数构造函数 @AllArgsConstructor @JsonIgnoreProperties(ignoreUnknown = true) // 用户详情信息 public class UserInfo { private long id; private String userMd; // 用户头像 private String userAvatar; // 用户标签 private String userTags; // 用户手机 private String phone; private String code; private String username; private String password; } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/entity/CommentEs.java ================================================ package pqdong.movie.recommend.data.entity; import com.fasterxml.jackson.annotation.JsonFormat; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import org.springframework.data.annotation.Id; import org.springframework.data.elasticsearch.annotations.DateFormat; import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.FieldType; import java.util.Date; /** * CommentEs * 评论 * @author pqdong * @since 2020/03/31 */ @Data @Builder @NoArgsConstructor @AllArgsConstructor @Document(indexName = "comment", type = "comment") public class CommentEs { @Id private Long commentId; // 用户id @Field(type= FieldType.Text) private String userMd; // 用户名称 @Field(type= FieldType.Text) private String userName; // 用户头像 @Field(type= FieldType.Text) private String userAvatar; // 电影id @Field(type= FieldType.Long) private Long movieId; // 电影名称 @Field(type= FieldType.Text) private String movieName; // 电影评论 @Field(type= FieldType.Text) private String content; // 赞同数 @Field(type= FieldType.Long) private Long votes; // 评论时间 @JsonFormat( pattern="yyyy-MM-dd HH:mm:ss",timezone = "GMT+8") @Field(format= DateFormat.custom,pattern = "yyyy-MM-dd HH:mm:ss",type= FieldType.Date) private Date commentTime; } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/entity/ConfigEntity.java ================================================ package pqdong.movie.recommend.data.entity; import lombok.Data; import lombok.NoArgsConstructor; import javax.persistence.*; /** * ConfigEntity * 配置实体 * @author pqdong * @since 2020/03/31 */ @Data @NoArgsConstructor @Entity @Table(name = "config") public class ConfigEntity { @Id @Column(name = "id") @GeneratedValue(strategy = GenerationType.IDENTITY) private long id; @Column(name = "key") private String key; @Column(name = "value") private String value; } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/entity/MovieEntity.java ================================================ package pqdong.movie.recommend.data.entity; /* * 电影 * @author pqdong */ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import javax.persistence.*; import java.util.Date; @Data @NoArgsConstructor @AllArgsConstructor @Table(name = "movie") @Entity public class MovieEntity { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "movie_id") private Long movieId; // 电影名称 @Column(name = "name") private String name; // 主演 @Column(name = "actors") private String actors; // 电影图片封面 @Column(name = "cover") private String cover; // 导演 @Column(name = "directors") private String directors; // 类型 @Column(name = "genres") private String genres; //播放地址 @Column(name = "official_site") private String officialSite; //电影制片地 @Column(name = "regions") private String regions; //电影语言 @Column(name = "languages") private String languages; //片长 @Column(name = "mins") private Integer mins; //评分 @Column(name = "score") private Float score; //投票数 @Column(name = "votes") private Integer votes; //标签 @Column(name = "tags") private String tags; //电影描述 @Column(name = "storyline") private String storyline; //年份 @Column(name = "year") private Integer year; //演员 @Column(name = "actor_ids") private String actorIds; //导演 @Column(name = "director_ids") private String directorIds; // 电影上映时间 @Temporal(TemporalType.DATE) @Column(name = "release_date") private Date releaseDate; } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/entity/MovieTagEntity.java ================================================ package pqdong.movie.recommend.data.entity; import lombok.Data; import lombok.NoArgsConstructor; import javax.persistence.*; @Data @NoArgsConstructor @Entity @Table(name = "movie_tags") public class MovieTagEntity { // 电影标签名称 @Id @Column(name = "movie_tag_id") @GeneratedValue(strategy = GenerationType.IDENTITY) private long id; @Column(name = "name") private String name; // 电影标签 @Column(name = "value") private String value; } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/entity/PersonEntity.java ================================================ package pqdong.movie.recommend.data.entity; import javax.persistence.*; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import java.sql.Timestamp; /** * 导演，演员 * @author pqdong */ @Data @NoArgsConstructor @AllArgsConstructor @Entity @Table(name = "person") public class PersonEntity { @Id @Column(name = "person_id") @GeneratedValue(strategy = GenerationType.IDENTITY) private long id; // 演员名称 @Column(name = "name") private String name; // 演员性别 @Column(name = "sex") private String sex; // 演员英文名称 @Column(name = "name_en") private String nameEn ; // 演员中文名称 @Column(name = "name_zn") private String nameZn ; // 出生日期 @Column(name = "birth") private String birth; // 出生地 @Column(name = "birth_place") private String birthPlace; // 星座 @Column(name = "constellation") private String constellation; // 职业 @Column(name = "profession") private String profession; // 简介 @Column(name = "biography") private String biography; // 头像 @Column(name = "avatar") private String avatar; } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/entity/RatingEntity.java ================================================ package pqdong.movie.recommend.data.entity; /* * 评分 * @author pqdong * @since 2020/04/06 */ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import org.springframework.context.annotation.Bean; import org.springframework.data.annotation.CreatedDate; import javax.persistence.*; import java.util.Date; @Data @NoArgsConstructor @AllArgsConstructor @Builder @Table(name = "rating") @Entity public class RatingEntity { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "rating_id") private Long ratingId; // 用户id @Column(name = "user_id") private Long userId; // 电影id @Column(name = "movie_id") private Long movieId; // 评分 @Column(name = "rating") private Integer rating; // 电影上映时间 @Temporal(TemporalType.DATE) @Column(name = "time") @CreatedDate private Date releaseDate; } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/entity/UserEntity.java ================================================ package pqdong.movie.recommend.data.entity; /* * 用户表 * @author pqdong * @since 2020/03/28 */ import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonNaming; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import lombok.Data; import lombok.NoArgsConstructor; import javax.persistence.*; import java.util.List; @Data @NoArgsConstructor @Entity @Table(name = "user") public class UserEntity { @Id @Column(name = "user_id") @GeneratedValue(strategy = GenerationType.IDENTITY) private long id; // 用户唯一标签，普通索引，md5值，长字符串为索引并不是很好 @Column(name = "user_md") private String userMd; // 用户昵称 @Column(name = "user_nickname") private String username; // 用户头像 @Column(name = "user_avatar") private String userAvatar; // 用户密码 @JsonIgnore @Column(name = "password") private String password; // 用户标签 @Column(name = "user_tags") private String userTags; // 用户手机号 @Column(name = "phone") private String phone; // 个人宣言 @Column(name = "motto") private String motto; @Column(name = "sex") private String sex; public List getFormatTag(){ return JSONObject.parseArray(this.userTags, String.class); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/repository/CommentEsRepo.java ================================================ package pqdong.movie.recommend.data.repository; import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; import org.springframework.stereotype.Repository; import pqdong.movie.recommend.data.entity.CommentEs; import java.util.List; @Repository public interface CommentEsRepo extends ElasticsearchRepository { List findByMovieId(Long movieId); List findByUserMd(String userMd); } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/repository/ConfigRepository.java ================================================ package pqdong.movie.recommend.data.repository; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import pqdong.movie.recommend.data.entity.ConfigEntity; /** * @author pqdong * @description * @date 2020-03-02 */ public interface ConfigRepository extends JpaRepository { @Query("SELECT e FROM ConfigEntity e WHERE e.key = :keys") ConfigEntity findConfigByKey(@Param("keys") String keys); } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/repository/MovieRepository.java ================================================ package pqdong.movie.recommend.data.repository; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import pqdong.movie.recommend.data.entity.MovieEntity; import pqdong.movie.recommend.data.entity.PersonEntity; import java.util.List; /** * @author pqdong * @description * @date 2020-03-02 */ public interface MovieRepository extends JpaRepository { @Query(nativeQuery = true, value = "select * from movie where 1=1 limit ?1") List findAllByCountLimit(@Param("num") int num); @Query(nativeQuery = true, value = "SELECT * FROM movie WHERE name LIKE CONCAT('%',?1,'%') limit ?2") List findAllByName(@Param("keys") String keys, @Param("total") int total); @Query(nativeQuery = true, value = "SELECT * FROM movie WHERE tags LIKE CONCAT('%',?1,'%') limit ?2") List findAllByTag(@Param("keys") String keys, @Param("total") int total); @Query(nativeQuery = true, value = "SELECT * FROM movie WHERE 1=1 ORDER BY score DESC limit 12") List findAllByHighScore(); @Query(nativeQuery = true, value = "SELECT * FROM movie WHERE actor_ids LIKE CONCAT('%',:keys,'%')") List findAllByPersonName(@Param("keys") String keys); @Query("SELECT e FROM MovieEntity e WHERE e.movieId = :movieId") MovieEntity findOneByMovieID(@Param("movieId") Long movieId); } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/repository/PersonRepository.java ================================================ package pqdong.movie.recommend.data.repository; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import pqdong.movie.recommend.data.entity.PersonEntity; import pqdong.movie.recommend.data.entity.UserEntity; import java.util.List; /** * @author pqdong * @description * @date 2020-03-02 */ public interface PersonRepository extends JpaRepository { @Query(nativeQuery = true, value = "select * from person where 1=1 limit ?1") List findAllByCountLimit(@Param("num") int num); @Query("SELECT e FROM PersonEntity e WHERE e.name like :keys") List findAllByName(@Param("keys") String keys); @Query("SELECT e FROM PersonEntity e WHERE e.id = :personId") PersonEntity findOneByPersonID(@Param("personId") long personId); } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/repository/RatingRepository.java ================================================ package pqdong.movie.recommend.data.repository; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import pqdong.movie.recommend.data.entity.MovieEntity; import pqdong.movie.recommend.data.entity.RatingEntity; import java.util.List; /** * @author pqdong * @description * @date 2020-03-02 */ public interface RatingRepository extends JpaRepository { @Query("SELECT e FROM RatingEntity e WHERE e.ratingId = :ratingId") RatingEntity findOneByRatingID(@Param("ratingId") Long ratingId); List findAllByUserId(Long userId); } ================================================ FILE: src/main/java/pqdong/movie/recommend/data/repository/UserRepository.java ================================================ package pqdong.movie.recommend.data.repository; import org.apache.hadoop.hdfs.server.namenode.BlockPlacementPolicyWithNodeGroup; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import pqdong.movie.recommend.data.entity.UserEntity; /** * @author pqdong * @description * @date 2020-03-03 */ public interface UserRepository extends JpaRepository { @Query("SELECT e FROM UserEntity e WHERE e.userMd = :userMd") UserEntity findByUserMd(@Param("userMd") String userMd); @Query("SELECT e FROM UserEntity e WHERE e.username = :userNickName") UserEntity findByUserNickName(@Param("userNickName") String userNickName); @Query("SELECT e FROM UserEntity e WHERE e.id = :userId") UserEntity findOneByUserID(@Param("userId") Long userId); } ================================================ FILE: src/main/java/pqdong/movie/recommend/domain/service/AsyncTask.java ================================================ package pqdong.movie.recommend.domain.service; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import java.util.Random; /** * AsyncTask * * @author pqdong * @description 作为一组领域对象，用以支持测试 * @since 2020/04/03 */ @Component @Slf4j public class AsyncTask{ private static Random random = new Random(); @Async("taskExecutor") public void doTaskOne() throws Exception { log.info("开始做任务一"); long start = System.currentTimeMillis(); try { Thread.sleep(random.nextInt(100)); }catch (InterruptedException e){ Thread.sleep(200); } long end = System.currentTimeMillis(); log.info("完成任务一，耗时：" + (end - start) + "毫秒"); } @Async("taskExecutor") public void doTaskTwo() throws Exception { log.info("开始做任务二"); long start = System.currentTimeMillis(); try { Thread.sleep(random.nextInt(100)); }catch (InterruptedException e){ Thread.sleep(200); } long end = System.currentTimeMillis(); log.info("完成任务二，耗时：" + (end - start) + "毫秒"); } @Async("taskExecutor") public void doTaskThree() throws Exception { log.info("开始做任务三"); long start = System.currentTimeMillis(); try { Thread.sleep(random.nextInt(100)); }catch (InterruptedException e){ Thread.sleep(200); } long end = System.currentTimeMillis(); log.info("完成任务三，耗时：" + (end - start) + "毫秒"); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/domain/service/MovieRecommender.java ================================================ package pqdong.movie.recommend.domain.service; import lombok.extern.slf4j.Slf4j; import org.apache.mahout.cf.taste.common.TasteException; import org.apache.mahout.cf.taste.impl.neighborhood.NearestNUserNeighborhood; import org.apache.mahout.cf.taste.impl.recommender.CachingRecommender; import org.apache.mahout.cf.taste.impl.recommender.GenericItemBasedRecommender; import org.apache.mahout.cf.taste.impl.recommender.GenericUserBasedRecommender; import org.apache.mahout.cf.taste.impl.similarity.EuclideanDistanceSimilarity; import org.apache.mahout.cf.taste.impl.similarity.PearsonCorrelationSimilarity; import org.apache.mahout.cf.taste.model.DataModel; import org.apache.mahout.cf.taste.recommender.RecommendedItem; import org.apache.mahout.cf.taste.recommender.Recommender; import org.apache.mahout.cf.taste.similarity.ItemSimilarity; import org.apache.mahout.cf.taste.similarity.UserSimilarity; import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.util.ArrayList; import java.util.List; @Component @Slf4j public class MovieRecommender { private final static int NEIGHBORHOOD_NUM = 3; @Resource(name = "mySQLDataModel") private DataModel dataModel; private List getRecommendedItemIDs(List recommendations){ List recommendItems = new ArrayList<>(); for(int i = 0 ; i < recommendations.size() ; i++) { RecommendedItem recommendedItem=recommendations.get(i); recommendItems.add(recommendedItem.getItemID()); } return recommendItems; } // 基于用户的推荐算法 public List userBasedRecommender(long userID,int size) throws TasteException { UserSimilarity similarity = new EuclideanDistanceSimilarity(dataModel ); NearestNUserNeighborhood neighbor = new NearestNUserNeighborhood(NEIGHBORHOOD_NUM, similarity, dataModel ); Recommender recommender = new CachingRecommender(new GenericUserBasedRecommender(dataModel , neighbor, similarity)); List recommendations = recommender.recommend(userID, size); return getRecommendedItemIDs(recommendations); } // 基于内容的推荐算法 public List itemBasedRecommender(long userID,int size) throws TasteException { ItemSimilarity itemSimilarity = new PearsonCorrelationSimilarity(dataModel); Recommender recommender = new GenericItemBasedRecommender(dataModel, itemSimilarity); List recommendations = recommender.recommend(userID, size); return getRecommendedItemIDs(recommendations); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/domain/util/ResponseMessage.java ================================================ package pqdong.movie.recommend.domain.util; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; /** * ResponseMessage * * @author pqdong * @since 2020/03/03 */ @Data @NoArgsConstructor @AllArgsConstructor public class ResponseMessage { private static final int SUCCESS_CODE = 0; private static final int FAILED_CODE = 1; private static final int PERMISSION_CODE = 2; private static final int ILLEGAL_CODE = 3; private static final int NEED_LOGIN_CODE = 10; private static final int ENCRYPTED_CODE = 1000; private int code; private String msg; private String description; private T data; public ResponseMessage(int code, String msg) { this.code = code; this.msg = msg; } public ResponseMessage(int code, String msg, String description) { this.code = code; this.msg = msg; this.description = description; } public static ResponseMessage successMessage(T data) { return new ResponseMessage<>(SUCCESS_CODE, "success", null, data); } public static ResponseMessage failedMessage(String message) { return new ResponseMessage<>(FAILED_CODE, "failed", message, null); } public static ResponseMessage failedMessage(String message, T data) { return new ResponseMessage<>(1, "failed", message, data); } public static ResponseMessage permissionMessage(T data) { return new ResponseMessage<>(PERMISSION_CODE, "permission denied", null, data); } public static ResponseMessage illegalMessage(String message) { return new ResponseMessage<>(ILLEGAL_CODE, "illegal", message, null); } public static ResponseMessage needLoginMessage() { return new ResponseMessage<>(NEED_LOGIN_CODE, "need login"); } public static ResponseMessage errorMessage(int code, String msg) { return new ResponseMessage<>(code, "error", msg, null); } public boolean success() { return this.code == SUCCESS_CODE; } } ================================================ FILE: src/main/java/pqdong/movie/recommend/exception/MyException.java ================================================ package pqdong.movie.recommend.exception; import lombok.Getter; import lombok.extern.slf4j.Slf4j; /** * MyException * * @author pqdong * @since 2020/03/04 */ @Slf4j public class MyException extends RuntimeException { @Getter private Integer code; public MyException(ResultEnum resultEnum) { super(resultEnum.getMsg()); this.code = resultEnum.getCode(); log.warn("exception! {}", resultEnum.getMsg()); } public MyException(Integer code, String msg) { super(msg); this.code = code; log.warn("exception! {}", msg); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/exception/ResultEnum.java ================================================ package pqdong.movie.recommend.exception; import lombok.Getter; /** * @author pqdong * @description * @date 2020/03/03 */ public enum ResultEnum { UNKNOW_ERROR(500, "服务器错误"), // 1开头为用户有关的错误 NEED_LOGIN(1001, "未登陆"), // 2开头为第三方接口的错误 SEND_NOTE_ERROR(2001, "发送短信失败"), QINIU_ERROR(2002, "七牛云接口出错"), SUCCESS(0, "成功"); @Getter private Integer code; @Getter private String msg; ResultEnum(Integer code, String msg) { this.code = code; this.msg = msg; } } ================================================ FILE: src/main/java/pqdong/movie/recommend/redis/CacheConfig.java ================================================ package pqdong.movie.recommend.redis; import lombok.extern.slf4j.Slf4j; import org.springframework.cache.annotation.CachingConfigurerSupport; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.data.redis.serializer.StringRedisSerializer; /** * CacheConfig * * @author pqdong * @since 2020/03/04 */ @Configuration @Slf4j public class CacheConfig extends CachingConfigurerSupport { @Bean public StringRedisTemplate stringRedisTemplate(LettuceConnectionFactory redisConnectionFactory) { StringRedisTemplate template = new StringRedisTemplate(); template.setKeySerializer(new PrefixRedisSerializer()); template.setValueSerializer(new StringRedisSerializer()); template.setConnectionFactory(redisConnectionFactory); template.setHashKeySerializer(new StringRedisSerializer()); template.setHashValueSerializer(new StringRedisSerializer()); template.afterPropertiesSet(); return template; } } ================================================ FILE: src/main/java/pqdong/movie/recommend/redis/PrefixRedisSerializer.java ================================================ package pqdong.movie.recommend.redis; import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.serializer.RedisSerializer; import org.springframework.util.Assert; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; /** * PrefixRedisSerializer * * @author pqdong * @since 2020/03/04 */ @Slf4j public class PrefixRedisSerializer implements RedisSerializer { private static String PREFIX = "movie"; private final Charset charset; public PrefixRedisSerializer() { this(StandardCharsets.UTF_8); } public PrefixRedisSerializer(Charset charset) { Assert.notNull(charset, "Charset must not be null!"); this.charset = charset; } public static void setPrefix(String prefix) { PREFIX = prefix; } @Override public String deserialize(byte[] bytes) { String saveKey = new String(bytes, charset); int indexOf = saveKey.indexOf(PREFIX); if (indexOf > 0) { log.warn("key缺少前缀"); } else { saveKey = saveKey.substring(indexOf + PREFIX.length() + 1); } return saveKey; } @Override public byte[] serialize(String string) { String key = PREFIX + ":" + string; return key.getBytes(charset); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/redis/RedisApi.java ================================================ /** * @(#)RedisService.java, 2019-03-12. * * Copyright 2019 Youdao, Inc. All rights reserved. * YOUDAO PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. */ package pqdong.movie.recommend.redis; import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.core.*; import org.springframework.data.redis.core.script.DefaultRedisScript; import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.util.*; import java.util.concurrent.TimeUnit; /** * RedisService * * @author pqdong * @since 2020/03/04 */ @Service @Slf4j public class RedisApi { @Resource(name = "stringRedisTemplate") private StringRedisTemplate redis; public boolean exist(String key) { try { return Objects.requireNonNull(redis.hasKey(key)); } catch (Exception e) { log.warn("redis exist error key={}", key, e); return false; } } public void expire(String key, long time, TimeUnit timeUnit) { try { redis.expire(key, time, timeUnit); } catch (Exception e) { log.warn("redis expire error key={}", key, e); } } public void delKey(String key) { try { redis.delete(key); } catch (Exception e) { log.warn("redis delKey error key={}", key, e); } } public void delKeys(List keys) { try { redis.delete(keys); } catch (Exception e) { log.warn("redis delKeys error keys={}", keys, e); } } public String getString(String key) { try { return redis.opsForValue().get(key); } catch (Exception e) { log.warn("redis getString error key={}", key, e); return null; } } public void delHashKey(String key, String name) { try { HashOperations opt = redis.opsForHash(); opt.delete(key, name); } catch (Exception e) { log.warn("redis delHashKey error key={}, name={}", key, name, e); } } public String hget(String key, String name) { try { HashOperations opt = redis.opsForHash(); return opt.get(key, name); } catch (Exception e) { log.warn("redis hget error key={}, name={}", key, name, e); return null; } } public Map hgetAll(String key) { try { HashOperations opt = redis.opsForHash(); return opt.entries(key); } catch (Exception e) { log.warn("redis hgetAll error key={}", key, e); return new HashMap<>(); } } public void hdel(String key, String field) { try { HashOperations opt = redis.opsForHash(); opt.delete(key, field); } catch (Exception e) { log.warn("redis hdel error key={}, field={}", key, field, e); } } public List hmget(String key, List hashKeys) { try { HashOperations opt = redis.opsForHash(); return opt.multiGet(key, hashKeys); } catch (Exception e) { log.warn("redis hmget error key={}, hashKeys={}", key, hashKeys, e); } return Collections.emptyList(); } public void hset(String key, String name, String value, long time, TimeUnit unit) { try { redis.executePipelined((RedisCallback) connection -> { HashOperations opt = redis.opsForHash(); connection.openPipeline(); opt.put(key, name, value); redis.expire(key, time, unit); connection.closePipeline(); return null; }); } catch (Exception e) { log.warn("redis hset error key={}, name={}, value={}", key, name, value, e); } } public void publish(String key, String message) { try { redis.convertAndSend(key, message); } catch (Exception e) { log.warn("redis publish error key={}, message={}", key, message, e); } } public void setValue(String key, String value, long time, TimeUnit unit) { try { redis.opsForValue().set(key, value, time, unit); } catch (Exception e) { log.warn("redis setValue error key={}, value={}", key, value, e); } } public void setValue(String key, String value, long time) { try { redis.executePipelined((RedisCallback) connection -> { connection.openPipeline(); redis.opsForValue().set(key, value); redis.expireAt(key, new Date(time)); connection.closePipeline(); return null; }); } catch (Exception e) { log.warn("redis setValue error key={}, value={}", key, value, e); } } } ================================================ FILE: src/main/java/pqdong/movie/recommend/redis/RedisKeys.java ================================================ package pqdong.movie.recommend.redis; /** * RedisKeys * * @author pqdong * @since 2020/03/04 */ public class RedisKeys { // user_token缓存 public static final String USER_TOKEN = "user_token"; // 高分电影缓存 public static final String HIGH_MOVIE = "high_movie"; // 评论防刷 public static final String BRUSH = "brush"; // 推荐电影 public static final String RECOMMEND = "recommend"; } ================================================ FILE: src/main/java/pqdong/movie/recommend/service/CommentService.java ================================================ package pqdong.movie.recommend.service; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.springframework.stereotype.Service; import pqdong.movie.recommend.data.dto.CommentSearchDto; import pqdong.movie.recommend.data.entity.CommentEs; import pqdong.movie.recommend.data.repository.CommentEsRepo; import pqdong.movie.recommend.redis.RedisApi; import pqdong.movie.recommend.redis.RedisKeys; import pqdong.movie.recommend.utils.RecommendUtils; import javax.annotation.Resource; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** * ConfigService * * @author pqdong * @since 2020/04/02 */ @Service @Slf4j public class CommentService { @Resource private CommentEsRepo commentEsRepo; @Resource private RedisApi redisApi; // 获取评论列表 public Map getCommentList(CommentSearchDto commentSearchDto) { Pair pair = RecommendUtils.getStartAndEnd(commentSearchDto.getPage(), commentSearchDto.getSize()); List allComment = getComments(commentSearchDto); List commentList = allComment.subList(pair.getLeft(), pair.getRight() <= allComment.size() ? pair.getRight() : allComment.size()); Map result = new HashMap<>(2, 1); result.put("total", commentList.size()); result.put("commentList", commentList.stream() .peek(m -> { if (StringUtils.isEmpty(m.getUserAvatar())) { m.setUserAvatar(RecommendUtils.getRandomAvatar(m.getUserAvatar())); } }) .collect(Collectors.toCollection(LinkedList::new))); return result; } // 根据条件搜索 private List getComments(CommentSearchDto commentSearchDto) { if (commentSearchDto.getMovieId() != null && commentSearchDto.getMovieId() != 0) { return commentEsRepo.findByMovieId(commentSearchDto.getMovieId()).stream() .sorted(Comparator.comparing(CommentEs::getCommentTime).reversed()) .collect(Collectors.toCollection(LinkedList::new)); }else if (!StringUtils.isEmpty(commentSearchDto.getUserMd())) { return commentEsRepo.findByUserMd(commentSearchDto.getUserMd()).stream() .sorted(Comparator.comparing(CommentEs::getCommentTime).reversed()) .collect(Collectors.toCollection(LinkedList::new)); } return new LinkedList<>(); } // 将评论信息写入es index public CommentEs submitComment(CommentEs commentEs) { // times 用来做防刷爬虫攻击，限制每个用户短时间内的第五次提交 Integer times = Integer.valueOf(Optional .ofNullable(redisApi.getString(RecommendUtils.getKey(RedisKeys.BRUSH, commentEs.getUserMd()))) .orElse("0")); if (times > 5){ return null; } else { times = times + 1; redisApi.setValue(RecommendUtils.getKey(RedisKeys.BRUSH, commentEs.getUserMd()), times.toString(), 3, TimeUnit.MINUTES); } return commentEsRepo.save(commentEs); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/service/ConfigService.java ================================================ package pqdong.movie.recommend.service; import org.springframework.stereotype.Service; import pqdong.movie.recommend.data.repository.ConfigRepository; import javax.annotation.Resource; /** * ConfigService * * @author pqdong * @since 2020/03/31 */ @Service public class ConfigService { @Resource private ConfigRepository configRepository; // 获取七牛云，阿里云短信 access_token等配置信息 public String getConfigValue(String key){ return configRepository.findConfigByKey(key).getValue(); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/service/ElasticSearchService.java ================================================ package pqdong.movie.recommend.service; import com.csvreader.CsvReader; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.PageRequest; import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; import org.springframework.data.elasticsearch.core.query.*; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import pqdong.movie.recommend.data.entity.CommentEs; import pqdong.movie.recommend.data.entity.MovieEntity; import pqdong.movie.recommend.data.entity.UserEntity; import pqdong.movie.recommend.data.repository.CommentEsRepo; import pqdong.movie.recommend.data.repository.MovieRepository; import pqdong.movie.recommend.data.repository.UserRepository; import pqdong.movie.recommend.utils.RecommendUtils; import javax.annotation.Resource; import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Optional; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; /** * ElasticSearchService * * @author pqdong * @since 2020/03/31 */ @Service @Slf4j public class ElasticSearchService { @Resource private ElasticsearchRestTemplate elasticsearchTemplate; @Resource private MovieRepository movieRepository; @Resource private UserRepository userRepository; @Resource private CommentEsRepo commentEsRepo; public List getAllIndex() { NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder().withQuery(matchAllQuery()) .withPageable(PageRequest.of(0, 100)); SearchQuery query = builder.build(); List index = elasticsearchTemplate.queryForList(query, CommentEs.class); log.info("{}", index); return index; } // 更新所有评论，异步处理 @Async("taskExecutor") public void updateAllComment(UserEntity userEntity) { List commentEs = commentEsRepo.findByUserMd(userEntity.getUserMd()); List queries = new ArrayList(); int counter = 0; for (CommentEs comment : commentEs) { comment.setUserAvatar(userEntity.getUserAvatar()); comment.setUserName(userEntity.getUsername()); IndexQuery indexQuery = new IndexQuery(); indexQuery.setId(Optional.ofNullable(comment.getCommentId()) .orElse(System.currentTimeMillis()) .toString()); try { indexQuery.setSource(new ObjectMapper().writeValueAsString(comment)); } catch (JsonProcessingException e) { log.info("{}", e.getMessage()); continue; } indexQuery.setIndexName("comment"); indexQuery.setType("comment"); queries.add(indexQuery); //分批提交修改 if (counter != 0 && counter % 1000 == 0) { elasticsearchTemplate.bulkIndex(queries); queries.clear(); } counter++; } // 提交不足量修改 if (queries.size() > 0) { elasticsearchTemplate.bulkIndex(queries); } if (counter > 0) { elasticsearchTemplate.refresh("comment"); } log.info("commentEs has update" + counter); } // 用于将csv文件中的数据导入到es表中，在处理用户昵称和电影名称时考虑到速度，不查询数据库，用现有数据代替 public long importCommentToEs() { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); try { ArrayList csvList = new ArrayList(); CsvReader reader = new CsvReader("D:\\graduation\\data\\moviedata\\comments.csv", ',', StandardCharsets.UTF_8); reader.readHeaders(); //跳过表头,不跳可以注释掉 while (reader.readRecord()) { csvList.add(reader.getValues()); //按行读取，并把每一行的数据添加到list集合 } reader.close(); List queries = new ArrayList(); int counter = 0; for (String[] comment : csvList) { IndexQuery indexQuery = new IndexQuery(); indexQuery.setId(comment[0]); indexQuery.setSource(new ObjectMapper().writeValueAsString(CommentEs.builder() .userAvatar(RecommendUtils.getRandomAvatar(comment[1])) .userMd(comment[1]) .userName(comment[1]) .commentTime(dateFormat.parse(comment[5])) .movieId(Long.valueOf(comment[2])) .content(comment[3]) .movieName(comment[2]) .votes(Long.valueOf(comment[4])) .build())); indexQuery.setIndexName("comment"); indexQuery.setType("comment"); queries.add(indexQuery); //分批提交修改 if (counter != 0 && counter % 1000 == 0) { elasticsearchTemplate.bulkIndex(queries); log.info("comment to es has update"); queries.clear(); } counter++; } // 提交不足量修改 if (queries.size() > 0) { elasticsearchTemplate.bulkIndex(queries); log.info("comment to es has update"); } if (counter > 0) { elasticsearchTemplate.refresh("comment"); log.info("comment to es has refresh"); } log.info("commentEs has update" + counter); return counter; } catch (Exception e) { log.info("{}", e.getMessage()); } return 0; } // 对电影下的评论数据进行数据处理 public long updateCommentToEs(Long movieId) { List movieEntities = new LinkedList<>(); if (movieId != null && movieId != 0) { movieEntities.add(movieRepository.findOneByMovieID(movieId)); } else { movieEntities = movieRepository.findAllByCountLimit(50); movieEntities.addAll(movieRepository.findAllByHighScore()); } List queries = new ArrayList(); int counter = 0; for (MovieEntity movieEntity : movieEntities) { List commentEs = commentEsRepo.findByMovieId(movieEntity.getMovieId()); for (CommentEs comment : commentEs) { UserEntity userEntity = userRepository.findByUserMd(comment.getUserMd()); if (userEntity == null) { if (comment.getContent().length()>5){ comment.setUserName(comment.getContent().substring(0,5)); }else{ comment.setUserName(comment.getContent()); } } else{ comment.setUserName(userEntity.getUsername()); } comment.setMovieName(movieEntity.getName()); IndexQuery indexQuery = new IndexQuery(); indexQuery.setId(Optional.ofNullable(comment.getCommentId()) .orElse(System.currentTimeMillis()) .toString()); try { indexQuery.setSource(new ObjectMapper().writeValueAsString(comment)); } catch (JsonProcessingException e) { log.info("{}", e.getMessage()); continue; } indexQuery.setIndexName("comment"); indexQuery.setType("comment"); queries.add(indexQuery); //分批提交修改 if (counter != 0 && counter % 1000 == 0) { elasticsearchTemplate.bulkIndex(queries); queries.clear(); } counter++; } } // 提交不足量修改 if (queries.size() > 0) { elasticsearchTemplate.bulkIndex(queries); } if (counter > 0) { elasticsearchTemplate.refresh("comment"); } log.info("commentEs has update" + counter); return counter; } } ================================================ FILE: src/main/java/pqdong/movie/recommend/service/MovieService.java ================================================ package pqdong.movie.recommend.service; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.mahout.cf.taste.common.TasteException; import org.springframework.stereotype.Service; import pqdong.movie.recommend.data.constant.ServerConstant; import pqdong.movie.recommend.data.dto.MovieSearchDto; import pqdong.movie.recommend.data.dto.RatingDto; import pqdong.movie.recommend.data.entity.MovieEntity; import pqdong.movie.recommend.data.entity.RatingEntity; import pqdong.movie.recommend.data.entity.UserEntity; import pqdong.movie.recommend.data.repository.MovieRepository; import pqdong.movie.recommend.data.repository.RatingRepository; import pqdong.movie.recommend.data.repository.UserRepository; import pqdong.movie.recommend.domain.service.MovieRecommender; import pqdong.movie.recommend.redis.RedisApi; import pqdong.movie.recommend.redis.RedisKeys; import pqdong.movie.recommend.utils.RecommendUtils; import javax.annotation.Resource; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** * MovieService * * @author pqdong * @since 2020/03/31 */ @Service @Slf4j public class MovieService { @Resource private ConfigService configService; @Resource private MovieRepository movieRepository; @Resource private RatingRepository ratingRepository; @Resource private UserRepository userRepository; @Resource private RedisApi redisApi; @Resource private MovieRecommender movieRecommender; private final static int RECOMMENT_SIZE = 4; // 获取推荐电影 public List getRecommendMovie(UserEntity user) { // 用户已经登录 List recommendMovies = new LinkedList<>(); String recommend = ""; if (user != null) { // load缓存数据 recommend = redisApi.getString(RecommendUtils.getKey(RedisKeys.RECOMMEND, user.getUserMd())); if (StringUtils.isEmpty(recommend)) { // 用户打过分 if (!ratingRepository.findAllByUserId(user.getId()).isEmpty()){ // 基于用户推荐 try { List movieIds = movieRecommender.itemBasedRecommender(user.getId(), RECOMMENT_SIZE); recommendMovies.addAll(movieRepository.findAllById(movieIds)); } catch (Exception e) { log.info("{}",e); } } } else { // 从缓存中直接加载 recommendMovies.addAll(JSONObject.parseArray(recommend, MovieEntity.class)); } } else{ // 用户未登录，随机返回 recommendMovies.addAll(movieRepository.findAllByCountLimit(RECOMMENT_SIZE)); } // 上述过程异常，或者用户未登录，直接根据标签查询数据库并推荐 if (recommendMovies.isEmpty() && user != null){ recommendMovies.addAll(movieRepository.findAllByTag(Optional.ofNullable(user.getFormatTag()) .orElse(Collections.singletonList(user.getUserTags())).get(0), RECOMMENT_SIZE)); } if (StringUtils.isEmpty(recommend)){ redisApi.setValue(RecommendUtils.getKey(RedisKeys.RECOMMEND, user.getUserMd()),JSONObject.toJSONString(recommendMovies),1,TimeUnit.DAYS ); } return recommendMovies; } // 获取高分电影 public Map getHighMovie() { String movieByRedis = redisApi.getString(RedisKeys.HIGH_MOVIE); Map result = new HashMap<>(2, 1); if (StringUtils.isEmpty(movieByRedis)) { List allMovies = movieRepository.findAllByHighScore(); redisApi.setValue(RedisKeys.HIGH_MOVIE, JSONObject.toJSONString(allMovies), 1, TimeUnit.DAYS); result.put("total", allMovies.size()); result.put("movieList", allMovies); } else { List allMovies = JSONObject.parseArray(movieByRedis, MovieEntity.class); result.put("total", allMovies.size()); result.put("movieList", allMovies); } return result; } // 获取电影标签 public List getMovieTags() { return JSON.parseArray(configService.getConfigValue("MOVIE_TAG"), String.class); } // 获取电影列表 public Map getAllMovie(String key, int page, int size) { Pair pair = RecommendUtils.getStartAndEnd(page, size); List allMovie = getMovies(key, "name", page * size); List movieList = allMovie.subList(pair.getLeft(), pair.getRight() <= allMovie.size() ? pair.getRight() : allMovie.size()); Map result = new HashMap<>(2, 1); result.put("total", movieList.size()); result.put("movieList", movieList.stream().peek(m -> { if (StringUtils.isEmpty(m.getCover())) { m.setCover(ServerConstant.DefaultImg); } }).collect(Collectors.toCollection(LinkedList::new))); return result; } // 根据条件搜索电影 public Map searchMovies(MovieSearchDto info) { Pair pair = RecommendUtils.getStartAndEnd(info.getPage(), info.getSize()); List allMovie = info.getTags().stream() .map(t -> getMovies(t, "tag", info.getPage() * info.getSize())) .flatMap(Collection::stream) .collect(Collectors.toCollection(LinkedList::new)); if (!StringUtils.isEmpty(info.getContent())) { allMovie.addAll(getMovies(info.getContent(), "name", info.getPage() * info.getSize())); } List movieList = allMovie.subList(pair.getLeft(), pair.getRight() <= allMovie.size() ? pair.getRight() : allMovie.size()); Map result = new HashMap<>(2, 1); result.put("total", movieList.size()); result.put("movieList", movieList.stream().peek(m -> { if (StringUtils.isEmpty(m.getCover())) { m.setCover(ServerConstant.DefaultImg); } }).collect(Collectors.toCollection(LinkedList::new))); return result; } // 根据电影名称等关键字获取电影列表 private List getMovies(String key, String type, int total) { if (StringUtils.isBlank(key)) { return movieRepository.findAllByCountLimit(total); } else if (type.equals("name")) { return movieRepository.findAllByName(key, total); } else { return movieRepository.findAllByTag(key, total); } } // 获取电影详情信息 public MovieEntity getMovie(Long movieId) { return movieRepository.findOneByMovieID(movieId); } // 获取演员所出演的所有电影 public List getPersonAttendMovie(String personName) { return movieRepository.findAllByPersonName(personName).stream().peek(g -> { if (StringUtils.isEmpty(g.getCover())) { g.setCover(ServerConstant.DefaultImg); } }).collect(Collectors.toCollection(LinkedList::new)); } // 对电影打分 public MovieEntity updateScore(RatingDto ratingDto) { UserEntity user = userRepository.findOneByUserID(ratingDto.getUserId()); MovieEntity movie = movieRepository.findOneByMovieID(ratingDto.getMovieId()); if (user == null){ return movie; } movie.setScore(ratingDto.getRating()); RatingEntity rating = RatingEntity.builder() .movieId(ratingDto.getMovieId()) .rating(ratingDto.getRating().intValue()) .userId(ratingDto.getUserId()) .releaseDate(new Date()) .build(); ratingRepository.save(rating); return movieRepository.save(movie); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/service/PersonService.java ================================================ package pqdong.movie.recommend.service; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.springframework.stereotype.Service; import pqdong.movie.recommend.data.constant.ServerConstant; import pqdong.movie.recommend.data.entity.PersonEntity; import pqdong.movie.recommend.data.repository.PersonRepository; import pqdong.movie.recommend.redis.RedisApi; import pqdong.movie.recommend.utils.RecommendUtils; import javax.annotation.Resource; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.stream.Collector; import java.util.stream.Collectors; /** * PersonService 演员相关 * * @author pqdong * @since 2020/03/31 */ @Service public class PersonService { @Resource private RedisApi redisApi; @Resource private PersonRepository personRepository; /** * @param key 查询关键字 * @param page 起始页码 * @param size 每页数据量 * @return 总数量和数据列表 * @method allPerson 获取 **/ public Map getAllPerson(String key, int page, int size) { Pair pair = RecommendUtils.getStartAndEnd(page, size); List allPerson = getPersons(key, page*size); List personList = allPerson.subList(pair.getLeft(), pair.getRight() <= allPerson.size() ? pair.getRight() : allPerson.size()); Map result = new HashMap<>(2, 1); result.put("total", personList.size()); result.put("personList", personList.stream().peek(p -> { if (StringUtils.isEmpty(p.getAvatar())){ p.setAvatar(ServerConstant.DefaultImg); } }).collect(Collectors.toCollection(LinkedList::new))); return result; } // 根据演员名称关键字等获取演员列表 private List getPersons(String key, int total) { List personList; if (StringUtils.isBlank(key)) { personList = personRepository.findAllByCountLimit(total); } else { personList = personRepository.findAllByName(key); } return personList; } // 获取导演，演员信息 public PersonEntity getPerson(Long personId){ return personRepository.findOneByPersonID(personId); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/service/QiNiuService.java ================================================ package pqdong.movie.recommend.service; import com.qiniu.common.Zone; import com.qiniu.http.Response; import com.qiniu.storage.Configuration; import com.qiniu.storage.UploadManager; import com.qiniu.util.Auth; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import pqdong.movie.recommend.exception.MyException; import pqdong.movie.recommend.exception.ResultEnum; /** * QiNiuService * * @author pqdong * @since 2020/03/04 */ @Service @Slf4j public class QiNiuService { @Autowired private ConfigService configService; /** * 七牛云 **/ private Auth auth = Auth.create("zfg7aGCs98DbKp_zKHzOAwzd6BoPhLjPkO5ohzEG", "l-fs00VcPP2nZRBKZmJj7LeSShi2wKxSMN5RL10w"); Configuration cfg = new Configuration(Zone.huadong()); private UploadManager uploadManager = new UploadManager(cfg); private String getUpToken() { return auth.uploadToken(configService.getConfigValue("BUCKET_NAME")); } public String uploadPicture(MultipartFile picture, String name) { try { Response response = uploadManager.put(picture.getBytes(), name, getUpToken()); if (response.isOK() && response.isJson()) { return configService.getConfigValue("QINIU_IMAGE_DOMAIN") + name; } } catch (Exception e) { log.warn("upload picture error", e); throw new MyException(ResultEnum.QINIU_ERROR); } return configService.getConfigValue("QINIU_IMAGE_DOMAIN") + name; } } ================================================ FILE: src/main/java/pqdong/movie/recommend/service/SmsService.java ================================================ package pqdong.movie.recommend.service; import com.aliyuncs.DefaultAcsClient; import com.aliyuncs.IAcsClient; import com.aliyuncs.dysmsapi.model.v20170525.SendSmsRequest; import com.aliyuncs.dysmsapi.model.v20170525.SendSmsResponse; import com.aliyuncs.exceptions.ClientException; import com.aliyuncs.http.MethodType; import com.aliyuncs.profile.DefaultProfile; import com.aliyuncs.profile.IClientProfile; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.springframework.stereotype.Service; import pqdong.movie.recommend.data.constant.UserConstant; import pqdong.movie.recommend.exception.MyException; import pqdong.movie.recommend.exception.ResultEnum; import pqdong.movie.recommend.redis.RedisApi; import pqdong.movie.recommend.utils.RecommendUtils; import javax.annotation.Resource; import java.util.concurrent.TimeUnit; /** * SmsService * * @author pqdong * @since 2020/03/04 */ @Service @Slf4j public class SmsService { @Resource private RedisApi redis; @Resource private ConfigService configService; /** * 阿里短信 **/ private IAcsClient getAcsClient() throws ClientException { IClientProfile profile = DefaultProfile.getProfile("cn-hangzhou", configService.getConfigValue("ACCESS_KEY_ID"), configService.getConfigValue("ACCESS_KEY_SECRET")); DefaultProfile.addEndpoint("cn-hangzhou", "cn-hangzhou", configService.getConfigValue("PRODUCT"), configService.getConfigValue("DOMAIN")); return new DefaultAcsClient(profile); } public String sendCode(String phone) { String randomCode = RandomStringUtils.randomNumeric(6); if (sendSms(phone, randomCode)) { // 有效期一天 redis.setValue(RecommendUtils.getKey(UserConstant.PHONE_CODE, phone), randomCode, 1, TimeUnit.DAYS); return randomCode; } return null; } private boolean sendSms(String phoneNumber, String randomCode) { SendSmsRequest request = new SendSmsRequest(); request.setMethod(MethodType.POST); request.setPhoneNumbers(phoneNumber); request.setSignName("花瓣电影"); request.setTemplateCode("SMS_185231476"); request.setTemplateParam("{\"code\":\"" + randomCode + "\"}"); try { SendSmsResponse sendSmsResponse = getAcsClient().getAcsResponse(request); if (sendSmsResponse.getCode() != null && sendSmsResponse.getCode().equals(UserConstant.OK)) { log.info("发送短信成功"); return true; } log.error(sendSmsResponse.getCode()); } catch (ClientException e) { log.error("ClientException异常：" + e.getMessage()); throw new MyException(ResultEnum.SEND_NOTE_ERROR); } log.error("发送短信失败"); return false; } } ================================================ FILE: src/main/java/pqdong/movie/recommend/service/UserService.java ================================================ package pqdong.movie.recommend.service; import io.micrometer.core.instrument.util.StringUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import pqdong.movie.recommend.data.constant.UserConstant; import pqdong.movie.recommend.data.entity.UserEntity; import pqdong.movie.recommend.data.repository.UserRepository; import pqdong.movie.recommend.data.dto.UserInfo; import pqdong.movie.recommend.redis.RedisApi; import pqdong.movie.recommend.redis.RedisKeys; import pqdong.movie.recommend.utils.Md5EncryptionHelper; import pqdong.movie.recommend.utils.RecommendUtils; import javax.annotation.Resource; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; /** * userService * * @author pqdong * @since 2020/03/03 */ @Slf4j @Service public class UserService { @Resource private UserRepository userRepository; @Resource private ElasticSearchService elasticSearchService; @Resource private QiNiuService qiNiuService; @Resource private RedisApi redisApi; public UserEntity updateUser(UserEntity user){ // 如果用户名已经存在，不进行更新 UserEntity userSearch = userRepository.findByUserNickName(user.getUsername()); // nickname唯一 if (null != userSearch && !userSearch.getUserMd().equals(user.getUserMd())){ return null; } UserEntity userInfo = userRepository.findByUserMd(user.getUserMd()); user.setPassword(userInfo.getPassword()); UserEntity userEntity = userRepository.save(user); if (userEntity != null){ elasticSearchService.updateAllComment(userEntity); } return userEntity; } public String register(UserInfo user){ String code = redisApi.getString(RecommendUtils.getKey(UserConstant.PHONE_CODE, user.getPhone())); if (StringUtils.isEmpty(code)){ return "验证码错误"; } else { if (!code.equals(user.getCode())){ return "验证码错误"; } UserEntity userEntity = userRepository.findByUserNickName(user.getUsername()); if (userEntity == null){ userEntity = new UserEntity(); try { BeanUtils.copyProperties(user,userEntity); } catch (Exception e){ log.error("copy properties error"); return "注册失败"; } userEntity.setPassword(Md5EncryptionHelper.getMD5WithSalt(user.getPassword())); userEntity.setUserMd(Md5EncryptionHelper.getMD5(Long.toString(System.currentTimeMillis()))); userEntity.setUserAvatar(RecommendUtils.getRandomAvatar(user.getUsername())); userRepository.save(userEntity); System.out.println(userEntity.toString()); return "success"; }else{ return "用户已经存在"; } } } public UserEntity getUserInfo(String token) { if (StringUtils.isEmpty(token)){ return null; } String userMd = redisApi.getString(RecommendUtils.getKey(RedisKeys.USER_TOKEN, token)); if (StringUtils.isEmpty(userMd)){ return null; } return userRepository.findByUserMd(userMd); } // 登录后需要前端设置header public Map login(String userName, String password) { UserEntity user = userRepository.findByUserNickName(userName); if (user == null){ return null; } if (user.getPassword().equals(Md5EncryptionHelper.getMD5WithSalt(password))){ Map info = new HashMap<>(); String token = RecommendUtils.genToken(); redisApi.setValue(RecommendUtils.getKey(RedisKeys.USER_TOKEN, token) , user.getUserMd(), 7, TimeUnit.DAYS); info.put("token", token); info.put("user", user); return info; } else { return null; } } // 上传并设置用户头像 public String uploadAvatar(String userMd, MultipartFile avatar) { String name = RecommendUtils.getKey(UserConstant.USER_AVATAR, userMd); String url = qiNiuService.uploadPicture(avatar, name); UserEntity entity = userRepository.findByUserMd(userMd); if (entity == null) { return "用户不存在"; } entity.setUserAvatar(url); UserEntity userEntity = userRepository.save(entity); if (userEntity != null){ elasticSearchService.updateAllComment(userEntity); } return url; } // 退出 public boolean logout(){ return true; } } ================================================ FILE: src/main/java/pqdong/movie/recommend/utils/LoginInterceptor.java ================================================ package pqdong.movie.recommend.utils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import org.springframework.web.method.HandlerMethod; import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; import pqdong.movie.recommend.annotation.LoginRequired; import pqdong.movie.recommend.data.constant.UserConstant; import pqdong.movie.recommend.data.entity.UserEntity; import pqdong.movie.recommend.data.repository.UserRepository; import pqdong.movie.recommend.exception.MyException; import pqdong.movie.recommend.exception.ResultEnum; import pqdong.movie.recommend.redis.RedisApi; import pqdong.movie.recommend.redis.RedisKeys; import javax.annotation.PostConstruct; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.lang.annotation.Annotation; import java.lang.reflect.Method; /** * LoginInterceptor * * @author pqdong * @since 2020/03/04 */ @Slf4j @Component public class LoginInterceptor extends HandlerInterceptorAdapter { private static LoginInterceptor loginInterceptor; @Resource private RedisApi redis; @Resource private UserRepository userRepository; @PostConstruct public void init() { loginInterceptor = this; } @Override public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { if (!(handler instanceof HandlerMethod)) { return true; } Method method = ((HandlerMethod) handler).getMethod(); // 判断需要调用需要登陆的接口时是否已经登陆 boolean isLoginRequired = isAnnotationPresent(method, LoginRequired.class); if (isLoginRequired) { String uri = request.getRequestURI(); String token = RecommendUtils.getToken(request); if (StringUtils.isEmpty(token)){ throw new MyException(ResultEnum.NEED_LOGIN); } String userMd = loginInterceptor.redis.getString(RecommendUtils.getKey(RedisKeys.USER_TOKEN, token)); if (StringUtils.isEmpty(userMd)){ // 没有获取到redis中的信息 throw new MyException(ResultEnum.NEED_LOGIN); } UserEntity user = loginInterceptor.userRepository.findByUserMd(userMd); if (user == null) { // token无法获取到用户信息代表未登陆 throw new MyException(ResultEnum.NEED_LOGIN); } // 退出时删除缓存 if (uri.contains(UserConstant.LOGOUT)) { loginInterceptor.redis.delKey(RecommendUtils.getKey(RedisKeys.USER_TOKEN, token)); } } return true; } private boolean isAnnotationPresent(Method method, Class annotationClass) { // 查找类注解或者方法注解 return method.getDeclaringClass().isAnnotationPresent(annotationClass) || method.isAnnotationPresent(annotationClass); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/utils/Md5EncryptionHelper.java ================================================ package pqdong.movie.recommend.utils; import lombok.AccessLevel; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; /** * MD5加密工具类 */ @Slf4j @NoArgsConstructor(access = AccessLevel.PRIVATE) public class Md5EncryptionHelper { /** * 获取MD5字符串 */ public static String getMD5(String content) { try { MessageDigest digest = MessageDigest.getInstance("MD5"); digest.update(content.getBytes()); return getHashString(digest); } catch (NoSuchAlgorithmException e) { e.printStackTrace(); } return null; } private static final String SALT = "0fpqd5e5a88bebae640a5daaa7c84734"; /** * 获取加盐的MD5字符串 */ public static String getMD5WithSalt(String content) { return getMD5(getMD5(content) + SALT); } private static String getHashString(MessageDigest digest) { StringBuilder builder = new StringBuilder(); for (byte b : digest.digest()) { builder.append(Integer.toHexString((b >> 4) & 0xf)); builder.append(Integer.toHexString(b & 0xf)); } return builder.toString(); } } ================================================ FILE: src/main/java/pqdong/movie/recommend/utils/RecommendUtils.java ================================================ package pqdong.movie.recommend.utils; import com.alibaba.fastjson.JSONObject; import com.google.common.base.Joiner; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.text.ParseException; import java.text.ParsePosition; import java.text.SimpleDateFormat; import java.util.*; import javax.servlet.http.HttpServletRequest; import lombok.AccessLevel; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import pqdong.movie.recommend.data.dto.UserInfo; /** * RecommendUtils * 工具类，单例模式 * @author pqdong * @since 2020/03/03 */ @Slf4j @NoArgsConstructor(access = AccessLevel.PRIVATE) public class RecommendUtils { public static final String SPLIT = ":"; public static final Joiner JOINER = Joiner.on(SPLIT); public static UserInfo getUser(HttpServletRequest request) { try { String userHeader = request.getHeader("user"); if (StringUtils.isNotEmpty(userHeader)) { // 中文需要解码 String user = URLDecoder.decode(userHeader, "UTF-8"); if (StringUtils.isNotEmpty(user)) { return JSONObject.parseObject(user, UserInfo.class); } } } catch (UnsupportedEncodingException e) { log.warn("getUser decode error"); } return null; } public static String getUserMd(HttpServletRequest request) { UserInfo user = getUser(request); if (user == null) { return null; } return user.getUserMd(); } /** * 生成token **/ public static String genToken() { return RandomStringUtils.randomAlphanumeric(10); } /** * 获取token */ public static String getToken(HttpServletRequest request) { try { String tokenHeader = request.getHeader("token"); if (StringUtils.isNotEmpty(tokenHeader)) { String token = URLDecoder.decode(tokenHeader, "UTF-8"); if (StringUtils.isNotEmpty(token)) { return token; } } } catch (UnsupportedEncodingException e) { log.warn("getToken decode error"); } return null; } public static String getKey(String... keys) { return JOINER.join(keys); } private static Random random = new Random(); /** * 生成随机过期时间，防止缓存雪崩 */ public static long genExpireTime(long time, int bound) { return time + random.nextInt(bound); } public static long genExpireTime(long time) { return genExpireTime(time, 3); } /** * 对Map按值排序,reverse为true代表从大到小排序，反之从小到大 * Map的value必须实现Comparable，否则无法实现排序 **/ @SuppressWarnings("unchecked") public static Map sortByValue(Map map, final boolean reverse) { List> list = new LinkedList<>(map.entrySet()); list.sort((o1, o2) -> { if (reverse) { return ((Comparable) o2.getValue()) .compareTo(o1.getValue()); } return ((Comparable) o1.getValue()) .compareTo(o2.getValue()); }); Map result = new LinkedHashMap<>(); for (Map.Entry entry : list) { result.put(entry.getKey(), entry.getValue()); } return result; } private static final String[] AVATARURLARRAY = new String[] { "http://oimagec6.ydstatic.com/image?id=-4541055657611236390&product=bisheng", "http://oimageb2.ydstatic.com/image?id=8981431901149412470&product=bisheng", "http://oimagea2.ydstatic.com/image?id=-6268572656325873060&product=bisheng", "http://oimagea2.ydstatic.com/image?id=-38385107928742692&product=bisheng", "http://oimageb4.ydstatic.com/image?id=3484504410139022595&product=bisheng" }; /** * 获取随机头像 **/ public static String getRandomAvatar(String userId) { int h = userId.hashCode(); h = h < 0 ? -h : h; return AVATARURLARRAY[h % AVATARURLARRAY.length]; } public static Pair getStartAndEnd(int page, int size) { int start = (page - 1) * size; int end = start + size; return Pair.of(start, end); } public static Date getFormatDate(Date time) throws ParseException { SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); if (time == null){ time = new Date(); } return formatter.parse(getFormatDateString(time)); } public static String getFormatDateString(Date time) { SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); return formatter.format(time); } } ================================================ FILE: src/main/resources/application.yml ================================================ server: port: 10015 max-http-header-size: 8192 tomcat: accesslog: enabled: true directory: access_log prefix: access_log pattern: '%h %l %u %t "%r" %s %b "%{Referer}i" "%{User-Agent}i" "%{X-Forwarded-For}i" "%{X-Real-IP}i" %D %S "%U" %v' basedir: ./ spring: application: name: movie-recommend datasource: driver-class-name: com.mysql.jdbc.Driver # mysql驱动 username: yourname # 你的数据库用户名 password: yourpassword # 你的数据库密码，url中添加数据库地址和端口 url: jdbc:mysql://yourip:port/movie?useSSL=false&serverTimezone=Asia/Shanghai&autoReconnect=true&failOverReadOnly=false redis: timeout: 300s # redis超时时间，建议用docker起一个redis容器即可 host: 127.0.0.1 port: 6379 jedis: pool: max-idle: 8 #最大连接数 max-active: 20 # 最大连接数,-1为不限制 min-idle: 0 # 最小空闲数量 max-wait: 1s # 最大阻塞等待时间，-1为不限制 jpa: database : MYSQL show-sql : true hibernate: ddl-auto: update naming: implicit-strategy: org.hibernate.boot.model.naming.ImplicitNamingStrategyLegacyJpaImpl physical-strategy: org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl properties: hibernate: show_sql: true globally_quoted_identifiers: true data: elasticsearch: repositories: enabled: true elasticsearch: ip: yourip rest: uris: yourip:9200 username: yourname password: yourpassword ================================================ FILE: src/test/java/pqdong/movie/recommond/BaseTest.java ================================================ /** * @(#)BaseTest.java, 2018-10-20. *

* Copyright 2018 Youdao, Inc. All rights reserved. * YOUDAO PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. */ package pqdong.movie.recommond; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit4.SpringRunner; import pqdong.movie.recommend.MovieRecommendApplication; /** * BaseTest * @author pqdong * @since 2020/04/03 */ @RunWith(SpringRunner.class) @SpringBootTest(classes = MovieRecommendApplication.class) public class BaseTest{ @Test public void test() { } } ================================================ FILE: src/test/java/pqdong/movie/recommond/UtilsTest.java ================================================ package pqdong.movie.recommond; import org.junit.Test; import pqdong.movie.recommend.utils.Md5EncryptionHelper; /** * UtilsTest * * @author pqdong * @since 2020/03/03 */ public class UtilsTest { @Test public void getMD5WithSalt() { System.out.println(Md5EncryptionHelper.getMD5WithSalt("123456")); } } ================================================ FILE: src/test/java/pqdong/movie/recommond/service/AsyncTaskTest.java ================================================ package pqdong.movie.recommond.service; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import pqdong.movie.recommend.domain.service.AsyncTask; import pqdong.movie.recommond.BaseTest; public class AsyncTaskTest extends BaseTest { @Autowired private AsyncTask asyncTask; @Test public void testAsyncTasks() throws Exception { asyncTask.doTaskOne(); asyncTask.doTaskTwo(); asyncTask.doTaskThree(); } } ================================================ FILE: start.sh ================================================ #!/bin/sh #预设java参数，自定义java参数用容器启动时配置环境变量JAVA_OPTS来实现，相同参数后者会覆盖前者 JAVA_DEFAULT_OPTS=" -javaagent:/skywalking-agent/skywalking-agent.jar -Dskywalking.collector.backend_service=${skywalking_service} -Dskywalking.agent.service_name=${HOSTNAME%-*-*} -Xms1g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=256m -XX:+PrintTenuringDistribution -XX:+PrintGCDetails -XX:+PrintHeapAtGC -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -Xloggc:gc-%t.log -Duser.timezone=GMT+08" #启动服务 exec java $JAVA_DEFAULT_OPTS $JAVA_OPTS -jar app.jar $JAVA_ARGS