Add gitlab ci support
authorChocobozzz <me@florianbigard.com>
Mon, 29 Jul 2019 09:59:29 +0000 (11:59 +0200)
committerChocobozzz <me@florianbigard.com>
Mon, 29 Jul 2019 11:40:39 +0000 (13:40 +0200)
41 files changed:
.gitlab-ci.yml [new file with mode: 0644]
.travis.yml
README.md
config/test.yaml
package.json
scripts/ci.sh [new file with mode: 0755]
scripts/clean/server/test.sh
scripts/nightly.sh [new file with mode: 0755]
scripts/test.sh
scripts/travis.sh [deleted file]
server/controllers/api/videos/abuse.ts
server/helpers/database-utils.ts
server/lib/activitypub/process/process-flag.ts
server/lib/activitypub/send/send-create.ts
server/lib/activitypub/send/send-delete.ts
server/lib/activitypub/send/send-flag.ts
server/lib/activitypub/send/send-follow.ts
server/lib/activitypub/send/send-undo.ts
server/lib/activitypub/send/send-update.ts
server/lib/activitypub/send/utils.ts
server/lib/job-queue/handlers/activitypub-follow.ts
server/tests/api/activitypub/fetch.ts
server/tests/api/ci-1.sh [new file with mode: 0644]
server/tests/api/ci-2.sh [new file with mode: 0644]
server/tests/api/ci-3.sh [new file with mode: 0644]
server/tests/api/ci-4.sh [new file with mode: 0644]
server/tests/api/notifications/user-notifications.ts
server/tests/api/server/email.ts
server/tests/api/server/handle-down.ts
server/tests/api/server/jobs.ts
server/tests/api/travis-1.sh [deleted file]
server/tests/api/travis-2.sh [deleted file]
server/tests/api/travis-3.sh [deleted file]
server/tests/api/travis-4.sh [deleted file]
server/tests/api/videos/video-schedule-update.ts
shared/extra-utils/miscs/miscs.ts
shared/extra-utils/miscs/sql.ts
shared/extra-utils/server/jobs.ts
shared/extra-utils/server/servers.ts
support/doc/development/release.md
yarn.lock

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644 (file)
index 0000000..2f69eb1
--- /dev/null
@@ -0,0 +1,100 @@
+image: chocobozzz/peertube-ci:10
+
+stages:
+  - build-and-lint
+  - test
+  - nightly
+
+before_script:
+  - 'sed -i -z "s/database:\n  hostname: ''localhost''/database:\n  hostname: ''postgres''/" config/test.yaml'
+  - 'sed -i -z "s/redis:\n  hostname: ''localhost''/redis:\n  hostname: ''redis''/" config/test.yaml'
+  - if [[ $CI_JOB_STAGE == "test" ]]; then psql -c "create user peertube with password 'peertube';"; fi
+  - NOCLIENT=1 yarn install --pure-lockfile --cache-folder .yarn-cache
+
+cache:
+  key: yarn
+  paths:
+    - .yarn-cache
+    - cached-fixtures
+
+###
+## Jobs templates
+#
+#.build-and-lint: &build-and-lint
+#  stage: build-and-lint
+#
+#.tests: &tests
+#  stage: test
+#  dependencies:
+#    - build-server
+#  services:
+#    - name: postgres:9.6
+#      alias: postgres
+#    - name: redis:latest
+#      alias: redis
+#  variables:
+#    PGHOST: postgres
+#    PGUSER: postgres
+#    REDIS_HOST: redis
+#  artifacts:
+#    expire_in: 1 day
+#    paths:
+#      - test*/logs
+#    when: always
+#
+####
+### Build and lint
+##
+#build-server:
+#  <<: *build-and-lint
+#  artifacts:
+#    expire_in: 5h
+#    paths:
+#      - dist/
+#  script:
+#    - npm run build:server
+#
+#lint:
+#  <<: *build-and-lint
+#  script:
+#    - yarn install --pure-lockfile --cache-folder .yarn-cache
+#    - npm run ci -- "lint"
+#
+####
+### Tests
+#
+#test-misc:
+#  <<: *tests
+#  script:
+#    - yarn install --pure-lockfile --cache-folder .yarn-cache
+#    - npm run ci -- "misc"
+#
+#test-cli:
+#  <<: *tests
+#  retry:
+#    max: 1
+#  script:
+#    - npm run ci -- "cli"
+#
+#api:
+#  <<: *tests
+#  parallel: 4
+#  retry:
+#    max: 1
+#  script:
+#    - NODE_PENDING_JOB_WAIT=1000 npm run ci -- api-$CI_NODE_INDEX
+
+build-nightly:
+  stage: nightly
+  only:
+    - schedules
+  script:
+    - yarn install --pure-lockfile --cache-folder .yarn-cache
+    - npm run nightly
+    - mkdir "${HOME}/.ssh"
+    - chmod 700 "${HOME}/.ssh"
+    - if [ ! -z ${DEPLOYEMENT_KNOWN_HOSTS+x} ]; then echo -e "${DEPLOYEMENT_KNOWN_HOSTS}" > ${HOME}/.ssh/known_hosts; fi
+    - eval `ssh-agent -s`
+    - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then ssh-add <(echo "${DEPLOYEMENT_KEY}"); fi
+    - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then scp ./peertube-nightly-* ${DEPLOYEMENT_USER}@${DEPLOYEMENT_HOST}:../../web/nightly; fi
+
index 9018b082a8986d138f4d9246d6e9fbc5526b045d..3d287a7828f52cd3b4b23db87f1e1344cb7e2fee 100644 (file)
@@ -47,7 +47,7 @@ matrix:
   - env: TEST_SUITE=lint
 
 script:
-  - NODE_PENDING_JOB_WAIT=2000 travis_retry npm run travis -- "$TEST_SUITE"
+  - NODE_PENDING_JOB_WAIT=2000 travis_retry npm run ci -- "$TEST_SUITE"
 
 after_failure:
   - cat test1/logs/peertube.log
index 3edaeef2f0e0b5ec811aff8b967102c0af140ba5..5de29a0678bf2288ff41dee5c118d1de294aec83 100644 (file)
--- a/README.md
+++ b/README.md
@@ -45,8 +45,8 @@ Be part of a network of multiple small federated, interoperable video hosting pr
 
   <br />
 
-  <a href="https://travis-ci.org/Chocobozzz/PeerTube">
-    <img src="https://travis-ci.org/Chocobozzz/PeerTube.svg?branch=develop" alt="Build Status" />
+  <a href="https://framagit.org/framasoft/peertube/PeerTube/commits/develop">
+    <img alt="pipeline status" src="https://framagit.org/framasoft/peertube/PeerTube/badges/develop/pipeline.svg" />
   </a>
 
   <a href="https://david-dm.org/Chocobozzz/PeerTube">
index e5ac74f57f14695e73a9f9d22c08730ad5a73b5a..8843bb2dc9f0a26fbc8f3483264fab481299f309 100644 (file)
@@ -20,6 +20,9 @@ database:
   hostname: 'localhost'
   port: 5432
 
+redis:
+  hostname: 'localhost'
+
 smtp:
   hostname: null
   port: 1025
index 300f54d9fea850f8f983dca3023c3bb41b839a33..e8821bc705a47ecd8813c12a51150c4c08245518 100644 (file)
     "postinstall": "test -n \"$NOCLIENT\" || (cd client && yarn install --pure-lockfile)",
     "tsc": "tsc",
     "commander": "commander",
-    "lint": "npm run travis -- lint",
+    "lint": "npm run ci -- lint",
     "ng": "ng",
     "nodemon": "nodemon",
     "ts-node": "ts-node",
     "tslint": "tslint",
     "concurrently": "concurrently",
+    "mocha-parallel-tests": "mocha-parallel-tests",
     "sasslint": "sass-lint --verbose --no-exit",
     "sasslint:fix": "sass-lint-auto-fix -c .sass-lint.yml --verbose",
     "mocha": "mocha",
-    "travis": "scripty",
+    "ci": "scripty",
     "release": "scripty",
+    "nightly": "scripty",
     "client-report": "scripty"
   },
   "husky": {
@@ -93,7 +95,7 @@
     ]
   },
   "resolutions": {
-    "@types/bluebird": "3.5.21"
+    "@types/bluebird": "3.5.27"
   },
   "dependencies": {
     "apicache": "^1.4.0",
     "maildev": "^1.0.0-rc3",
     "marked-man": "^0.6.0",
     "mocha": "^6.0.0",
-    "mocha-parallel-tests": "^2.1.0",
+    "mocha-parallel-tests": "^2.2.1",
     "nodemon": "^1.18.6",
     "sass-lint": "^1.12.1",
     "source-map-support": "^0.5.0",
diff --git a/scripts/ci.sh b/scripts/ci.sh
new file mode 100755 (executable)
index 0000000..1819f73
--- /dev/null
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+set -eu
+
+if [ $# -eq 0 ]; then
+    echo "Need test suite argument."
+    exit -1
+fi
+
+killall -q peertube || true
+
+if [ "$1" = "misc" ]; then
+    npm run build -- --light-fr
+    mocha --timeout 5000 --exit --require ts-node/register --bail server/tests/client.ts \
+        server/tests/feeds/index.ts \
+        server/tests/misc-endpoints.ts \
+        server/tests/helpers/index.ts \
+        server/tests/plugins/index.ts
+elif [ "$1" = "cli" ]; then
+    npm run build:server
+    CC=gcc-4.9 CXX=g++-4.9 npm run setup:cli
+    mocha --timeout 5000 --exit --require ts-node/register --bail server/tests/cli/index.ts
+elif [ "$1" = "api-1" ]; then
+    npm run build:server
+    sh ./server/tests/api/ci-1.sh 2
+elif [ "$1" = "api-2" ]; then
+    npm run build:server
+    sh ./server/tests/api/ci-2.sh 2
+elif [ "$1" = "api-3" ]; then
+    npm run build:server
+    sh ./server/tests/api/ci-3.sh 2
+elif [ "$1" = "api-4" ]; then
+    npm run build:server
+    sh ./server/tests/api/ci-4.sh 2
+elif [ "$1" = "lint" ]; then
+    npm run tslint -- --project ./tsconfig.json -c ./tslint.json server.ts "server/**/*.ts" "shared/**/*.ts"
+
+    ( cd client
+      npm run lint
+    )
+fi
index 20745170dc3e0ebc47aae7ca6395eb15a7065b48..dbd399aaa5172a89981ca606e885dade2a3133bc 100755 (executable)
@@ -5,7 +5,7 @@ set -eu
 recreateDB () {
   dbname="peertube_test$1"
 
-  dropdb --if-exists "$dbname"
+  dropdb --if-exists "$dbname" 2>&1
 
   createdb -O peertube "$dbname"
   psql -c "CREATE EXTENSION pg_trgm;" "$dbname" &
@@ -18,10 +18,15 @@ removeFiles () {
 
 dropRedis () {
   port=$((9000+$1))
+  host="localhost"
 
-  redis-cli KEYS "bull-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
-  redis-cli KEYS "redis-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
-  redis-cli KEYS "*redis-localhost:$port-" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
+  if [ ! -z ${GITLAB_CI+x} ]; then
+    host="redis"
+  fi
+
+  redis-cli -h "$host" KEYS "bull-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli -h "$host" DEL
+  redis-cli -h "$host" KEYS "redis-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli -h "$host" DEL
+  redis-cli -h "$host" KEYS "*redis-localhost:$port-" | grep -v empty | xargs --no-run-if-empty redis-cli -h "$host" DEL
 }
 
 seq=$(seq 1 6)
diff --git a/scripts/nightly.sh b/scripts/nightly.sh
new file mode 100755 (executable)
index 0000000..bde5d57
--- /dev/null
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -eu
+
+shutdown() {
+  # Get our process group id
+  # shellcheck disable=SC2009
+  PGID=$(ps -o pgid= $$ | grep -o "[0-9]*")
+
+  # Kill it in a new new process group
+  setsid kill -- -"$PGID"
+  exit 0
+}
+
+trap "shutdown" SIGINT SIGTERM
+
+today=$(date '+%F')
+directory_name="peertube-nightly-$today"
+tar_name="peertube-nightly-$today.tar.xz"
+
+npm run build
+
+# Creating the archives
+(
+  # local variables
+  directories_to_archive=("$directory_name/CREDITS.md" "$directory_name/FAQ.md" \
+                          "$directory_name/LICENSE" "$directory_name/README.md" \
+                          "$directory_name/client/dist/" "$directory_name/client/yarn.lock" \
+                          "$directory_name/client/package.json" "$directory_name/config" \
+                          "$directory_name/dist" "$directory_name/package.json" \
+                          "$directory_name/scripts" "$directory_name/support" \
+                          "$directory_name/tsconfig.json" "$directory_name/yarn.lock")
+
+  # temporary setup
+  cd ..
+  ln -s "PeerTube" "$directory_name"
+
+  XZ_OPT=-e9 tar cfJ "PeerTube/$tar_name" "${directories_to_archive[@]}"
+
+  # temporary setup destruction
+  rm "$directory_name"
+)
index 5ec7a592059d16905603989a29f3484a9ff044ab..1b6b29b66ec3184a7152a1d0bdb6bd95b370ca5e 100755 (executable)
@@ -5,6 +5,6 @@ set -eu
 npm run build:server
 npm run setup:cli
 
-npm run travis -- lint
+npm run ci -- lint
 
 mocha --exit --require ts-node/register/type-check --bail server/tests/index.ts
diff --git a/scripts/travis.sh b/scripts/travis.sh
deleted file mode 100755 (executable)
index 42e2329..0000000
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-if [ $# -eq 0 ]; then
-    echo "Need test suite argument."
-    exit -1
-fi
-
-killall -q peertube || true
-
-if [ "$1" = "misc" ]; then
-    npm run build -- --light-fr
-    mocha --timeout 5000 --exit --require ts-node/register --bail server/tests/client.ts \
-        server/tests/feeds/index.ts \
-        server/tests/misc-endpoints.ts \
-        server/tests/helpers/index.ts \
-        server/tests/plugins/index.ts
-elif [ "$1" = "cli" ]; then
-    npm run build:server
-    CC=gcc-4.9 CXX=g++-4.9 npm run setup:cli
-    mocha --timeout 5000 --exit --require ts-node/register --bail server/tests/cli/index.ts
-elif [ "$1" = "api-1" ]; then
-    npm run build:server
-    sh ./server/tests/api/travis-1.sh 2
-elif [ "$1" = "api-2" ]; then
-    npm run build:server
-    sh ./server/tests/api/travis-2.sh 2
-elif [ "$1" = "api-3" ]; then
-    npm run build:server
-    sh ./server/tests/api/travis-3.sh 2
-elif [ "$1" = "api-4" ]; then
-    npm run build:server
-    sh ./server/tests/api/travis-4.sh 2
-elif [ "$1" = "lint" ]; then
-    npm run tslint -- --project ./tsconfig.json -c ./tslint.json server.ts "server/**/*.ts" "shared/**/*.ts"
-
-    ( cd client
-      npm run lint
-    )
-fi
index ca70230a2f475eb05e7a7f40c9821a62fdf15b12..77808466c3cc8128b33423d89af40c77dc3171fe 100644 (file)
@@ -113,16 +113,16 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) {
 
     // We send the video abuse to the origin server
     if (videoInstance.isOwned() === false) {
-      await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance)
+      await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t)
     }
 
-    Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance)
-
     auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON()))
 
     return videoAbuseInstance
   })
 
+  Notifier.Instance.notifyOnNewVideoAbuse(videoAbuse)
+
   logger.info('Abuse report for video %s created.', videoInstance.name)
 
   return res.json({ videoAbuse: videoAbuse.toFormattedJSON() }).end()
index 39c74b2fdec9d87e791abb3de9be7b44dbc19083..6c5068fb0d631622d21e4fab74f6262dde5d1a2d 100644 (file)
@@ -2,6 +2,7 @@ import * as retry from 'async/retry'
 import * as Bluebird from 'bluebird'
 import { Model } from 'sequelize-typescript'
 import { logger } from './logger'
+import { Transaction } from 'sequelize'
 
 function retryTransactionWrapper <T, A, B, C> (
   functionToRetry: (arg1: A, arg2: B, arg3: C) => Promise<T> | Bluebird<T>,
@@ -72,11 +73,18 @@ function resetSequelizeInstance (instance: Model<any>, savedFields: object) {
   })
 }
 
+function afterCommitIfTransaction (t: Transaction, fn: Function) {
+  if (t) return t.afterCommit(() => fn())
+
+  return fn()
+}
+
 // ---------------------------------------------------------------------------
 
 export {
   resetSequelizeInstance,
   retryTransactionWrapper,
   transactionRetryer,
-  updateInstanceWithAnother
+  updateInstanceWithAnother,
+  afterCommitIfTransaction
 }
index 0b39760893b106802fe40e05ba2cbf1f47a131ba..8faab051eaf50c80991772d68707a717e566b6a0 100644 (file)
@@ -31,7 +31,7 @@ async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag,
 
   const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: flag.object })
 
-  return sequelizeTypescript.transaction(async t => {
+  const videoAbuse = await sequelizeTypescript.transaction(async t => {
     const videoAbuseData = {
       reporterAccountId: account.id,
       reason: flag.content,
@@ -42,8 +42,10 @@ async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag,
     const videoAbuseInstance = await VideoAbuseModel.create(videoAbuseData, { transaction: t })
     videoAbuseInstance.Video = video
 
-    Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance)
-
     logger.info('Remote abuse for video uuid %s created', flag.object)
+
+    return videoAbuseInstance
   })
+
+  Notifier.Instance.notifyOnNewVideoAbuse(videoAbuse)
 }
index 28f18595ba485a76793a19e8d0cb84062dfa5e97..9c21149f20c184b34febb92f89f8029bf8b8c1ad 100644 (file)
@@ -11,6 +11,7 @@ import { VideoRedundancyModel } from '../../../models/redundancy/video-redundanc
 import { VideoPlaylistModel } from '../../../models/video/video-playlist'
 import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
 import { getServerActor } from '../../../helpers/utils'
+import * as Bluebird from 'bluebird'
 
 async function sendCreateVideo (video: VideoModel, t: Transaction) {
   if (video.privacy === VideoPrivacy.PRIVATE) return undefined
@@ -82,7 +83,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
 
   // This was a reply, send it to the parent actors
   const actorsException = [ byActor ]
-  await broadcastToActors(createActivity, byActor, parentsCommentActors, actorsException)
+  await broadcastToActors(createActivity, byActor, parentsCommentActors, t, actorsException)
 
   // Broadcast to our followers
   await broadcastToFollowers(createActivity, byActor, [ byActor ], t)
@@ -91,7 +92,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
   if (isOrigin) return broadcastToFollowers(createActivity, byActor, actorsInvolvedInComment, t, actorsException)
 
   // Send to origin
-  return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
+  t.afterCommit(() => unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl))
 }
 
 function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {
index 7bf5ca520f9165cb46c051d1eafed61f2a222f21..7a1d6f0ba8da491cf094d097b7d5668d79034685 100644 (file)
@@ -59,7 +59,7 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans
 
   // This was a reply, send it to the parent actors
   const actorsException = [ byActor ]
-  await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), actorsException)
+  await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), t, actorsException)
 
   // Broadcast to our followers
   await broadcastToFollowers(activity, byActor, [ byActor ], t)
@@ -68,7 +68,7 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans
   if (isVideoOrigin) return broadcastToFollowers(activity, byActor, actorsInvolvedInComment, t, actorsException)
 
   // Send to origin
-  return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
+  t.afterCommit(() => unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl))
 }
 
 async function sendDeleteVideoPlaylist (videoPlaylist: VideoPlaylistModel, t: Transaction) {
index 96a7311b9acb7ecf0a41908d9ea7c2c484b7df2c..61ee389a61157a84cee8d35ed9c1ad9f95e7f62f 100644 (file)
@@ -6,8 +6,9 @@ import { unicastTo } from './utils'
 import { logger } from '../../../helpers/logger'
 import { ActivityAudience, ActivityFlag } from '../../../../shared/models/activitypub'
 import { audiencify, getAudience } from '../audience'
+import { Transaction } from 'sequelize'
 
-async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) {
+async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel, t: Transaction) {
   if (!video.VideoChannel.Account.Actor.serverId) return // Local user
 
   const url = getVideoAbuseActivityPubUrl(videoAbuse)
@@ -18,7 +19,7 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
   const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
   const flagActivity = buildFlagActivity(url, byActor, videoAbuse, audience)
 
-  return unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+  t.afterCommit(() => unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl))
 }
 
 function buildFlagActivity (url: string, byActor: ActorModel, videoAbuse: VideoAbuseModel, audience: ActivityAudience): ActivityFlag {
index 2c3d02014e9022a35b781f58246872359ed52499..c6e7fe83d81946f8af1f608a238bb1ebbfc9d3cd 100644 (file)
@@ -4,8 +4,9 @@ import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { getActorFollowActivityPubUrl } from '../url'
 import { unicastTo } from './utils'
 import { logger } from '../../../helpers/logger'
+import { Transaction } from 'sequelize'
 
-function sendFollow (actorFollow: ActorFollowModel) {
+function sendFollow (actorFollow: ActorFollowModel, t: Transaction) {
   const me = actorFollow.ActorFollower
   const following = actorFollow.ActorFollowing
 
@@ -17,7 +18,7 @@ function sendFollow (actorFollow: ActorFollowModel) {
   const url = getActorFollowActivityPubUrl(me, following)
   const data = buildFollowActivity(url, me, following)
 
-  return unicastTo(data, me, following.inboxUrl)
+  t.afterCommit(() => unicastTo(data, me, following.inboxUrl))
 }
 
 function buildFollowActivity (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow {
index 8727a121e786751a54713c48569d105118bc0859..8fcbbac5c95a94a0dfa35e96d38f4607a078c9a4 100644 (file)
@@ -37,7 +37,7 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
   const followActivity = buildFollowActivity(followUrl, me, following)
   const undoActivity = undoActivityData(undoUrl, me, followActivity)
 
-  return unicastTo(undoActivity, me, following.inboxUrl)
+  t.afterCommit(() => unicastTo(undoActivity, me, following.inboxUrl))
 }
 
 async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
index 7411c08d578d92e4712381028a1c661451169092..5bf092894eba54a65b5d6548712f018f511b444b 100644 (file)
@@ -26,7 +26,9 @@ async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByAct
   const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString())
 
   // Needed to build the AP object
-  if (!video.VideoCaptions) video.VideoCaptions = await video.$get('VideoCaptions') as VideoCaptionModel[]
+  if (!video.VideoCaptions) {
+    video.VideoCaptions = await video.$get('VideoCaptions', { transaction: t }) as VideoCaptionModel[]
+  }
 
   const videoObject = video.toActivityPubObject()
   const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)
index 69706e620ebcd180a85f7b7a3a6a9e51ee6cdd87..1faae1d84fe986d9acf644c76c680d96e20dcda4 100644 (file)
@@ -7,6 +7,7 @@ import { JobQueue } from '../../job-queue'
 import { VideoModel } from '../../../models/video/video'
 import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience'
 import { getServerActor } from '../../../helpers/utils'
+import { afterCommitIfTransaction } from '../../../helpers/database-utils'
 
 async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: {
   byActor: ActorModel,
@@ -20,7 +21,9 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud
     const audience = getRemoteVideoAudience(options.video, actorsInvolvedInVideo)
     const activity = activityBuilder(audience)
 
-    return unicastTo(activity, options.byActor, options.video.VideoChannel.Account.Actor.sharedInboxUrl)
+    return afterCommitIfTransaction(options.transaction, () => {
+      return unicastTo(activity, options.byActor, options.video.VideoChannel.Account.Actor.sharedInboxUrl)
+    })
   }
 
   // Send to followers
@@ -28,6 +31,7 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud
   const activity = activityBuilder(audience)
 
   const actorsException = [ options.byActor ]
+
   return broadcastToFollowers(activity, options.byActor, actorsInvolvedInVideo, options.transaction, actorsException)
 }
 
@@ -76,7 +80,7 @@ async function forwardActivity (
     uris,
     body: activity
   }
-  return JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload })
+  return afterCommitIfTransaction(t, () => JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload }))
 }
 
 async function broadcastToFollowers (
@@ -87,20 +91,22 @@ async function broadcastToFollowers (
   actorsException: ActorModel[] = []
 ) {
   const uris = await computeFollowerUris(toFollowersOf, actorsException, t)
-  return broadcastTo(uris, data, byActor)
+
+  return afterCommitIfTransaction(t, () => broadcastTo(uris, data, byActor))
 }
 
 async function broadcastToActors (
   data: any,
   byActor: ActorModel,
   toActors: ActorModel[],
+  t?: Transaction,
   actorsException: ActorModel[] = []
 ) {
   const uris = await computeUris(toActors, actorsException)
-  return broadcastTo(uris, data, byActor)
+  return afterCommitIfTransaction(t, () => broadcastTo(uris, data, byActor))
 }
 
-async function broadcastTo (uris: string[], data: any, byActor: ActorModel) {
+function broadcastTo (uris: string[], data: any, byActor: ActorModel) {
   if (uris.length === 0) return undefined
 
   logger.debug('Creating broadcast job.', { uris })
@@ -114,7 +120,7 @@ async function broadcastTo (uris: string[], data: any, byActor: ActorModel) {
   return JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload })
 }
 
-async function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) {
+function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) {
   logger.debug('Creating unicast job.', { uri: toActorUrl })
 
   const payload = {
@@ -123,7 +129,7 @@ async function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) {
     body: data
   }
 
-  return JobQueue.Instance.createJob({ type: 'activitypub-http-unicast', payload })
+  JobQueue.Instance.createJob({ type: 'activitypub-http-unicast', payload })
 }
 
 // ---------------------------------------------------------------------------
index b3defb6173f91fc26528a0a8a722a8daa9fc9c74..4ae66cd0165de75fce6cf939ed1b1028d8563a1e 100644 (file)
@@ -69,7 +69,7 @@ async function follow (fromActor: ActorModel, targetActor: ActorModel) {
     actorFollow.ActorFollower = fromActor
 
     // Send a notification to remote server if our follow is not already accepted
-    if (actorFollow.state !== 'accepted') await sendFollow(actorFollow)
+    if (actorFollow.state !== 'accepted') sendFollow(actorFollow, t)
 
     return actorFollow
   })
index 3a1c0d321272685c9848908a311dcc6724e42005..3d54c20428e1de58d47a80d1c1bf31f6f938a3c8 100644 (file)
@@ -87,7 +87,7 @@ describe('Test ActivityPub fetcher', function () {
   })
 
   after(async function () {
-    this.timeout(10000)
+    this.timeout(20000)
 
     await cleanupTests(servers)
 
diff --git a/server/tests/api/ci-1.sh b/server/tests/api/ci-1.sh
new file mode 100644 (file)
index 0000000..8dd9756
--- /dev/null
@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+
+set -eu
+
+checkParamFiles=$(find server/tests/api/check-params -type f | grep -v index.ts | xargs echo)
+notificationsFiles=$(find server/tests/api/notifications -type f | grep -v index.ts | xargs echo)
+searchFiles=$(find server/tests/api/search -type f | grep -v index.ts | xargs echo)
+
+MOCHA_PARALLEL=true npm run mocha --timeout 30000 --exit --require ts-node/register --bail \
+    $notificationsFiles $searchFiles $checkParamFiles
diff --git a/server/tests/api/ci-2.sh b/server/tests/api/ci-2.sh
new file mode 100644 (file)
index 0000000..16ab585
--- /dev/null
@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+
+set -eu
+
+serverFiles=$(find server/tests/api/server -type f | grep -v index.ts | xargs echo)
+usersFiles=$(find server/tests/api/users -type f | grep -v index.ts | xargs echo)
+
+MOCHA_PARALLEL=true npm run mocha-parallel-tests -- --max-parallel $1 --timeout 30000 --exit --require ts-node/register --bail \
+    $serverFiles $usersFiles
diff --git a/server/tests/api/ci-3.sh b/server/tests/api/ci-3.sh
new file mode 100644 (file)
index 0000000..fc96f6f
--- /dev/null
@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+
+set -eu
+
+videosFiles=$(find server/tests/api/videos -type f | grep -v index.ts | xargs echo)
+
+MOCHA_PARALLEL=true npm run mocha --timeout 30000 --exit --require ts-node/register --bail \
+    $videosFiles
diff --git a/server/tests/api/ci-4.sh b/server/tests/api/ci-4.sh
new file mode 100644 (file)
index 0000000..74809e1
--- /dev/null
@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+
+set -eu
+
+redundancyFiles=$(find server/tests/api/redundancy -type f | grep -v index.ts | xargs echo)
+activitypubFiles=$(find server/tests/api/activitypub -type f | grep -v index.ts | xargs echo)
+
+MOCHA_PARALLEL=true npm run mocha-parallel-tests -- --max-parallel $1 --timeout 30000 --exit --require ts-node/register --bail \
+    $redundancyFiles $activitypubFiles
index 662b64e05467f4ea0a9900eacf40f15c5e60d020..6fa6305623486d72b2090bea6d34584afaced017 100644 (file)
@@ -782,7 +782,7 @@ describe('Test users notifications', function () {
     it('Should not send a notification before the video is published', async function () {
       this.timeout(20000)
 
-      let updateAt = new Date(new Date().getTime() + 100000)
+      let updateAt = new Date(new Date().getTime() + 1000000)
 
       const data = {
         privacy: VideoPrivacy.PRIVATE,
@@ -1074,7 +1074,7 @@ describe('Test users notifications', function () {
     it('Should send unblacklist but not published/subscription notes after unblacklisted if scheduled update pending', async function () {
       this.timeout(20000)
 
-      let updateAt = new Date(new Date().getTime() + 100000)
+      let updateAt = new Date(new Date().getTime() + 1000000)
 
       const name = 'video with auto-blacklist and future schedule ' + uuidv4()
 
index 7b7acfd12a161f31b71590dccd64ccc45dfb2880..c55a221f2524f1a58eba107e23341e50f58cfd41 100644 (file)
@@ -40,7 +40,7 @@ describe('Test emails', function () {
   let emailPort: number
 
   before(async function () {
-    this.timeout(30000)
+    this.timeout(50000)
 
     emailPort = await MockSmtpServer.Instance.collectEmails(emails)
 
index 068654d8c4f3906b73c4ce21bed57b6b013e8f3f..a225443c5bf0ad8327debe15dfd2fc973565197a 100644 (file)
@@ -109,7 +109,7 @@ describe('Test handle downs', function () {
   })
 
   it('Should remove followers that are often down', async function () {
-    this.timeout(60000)
+    this.timeout(240000)
 
     // Server 2 and 3 follow server 1
     await follow(servers[1].url, [ servers[0].url ], servers[1].accessToken)
index 3ab2fe1202aca5bff739fa7058113c9c5c482a77..ceea47a854a77b3377b1e1cd3bd067fee7ed8787 100644 (file)
@@ -8,6 +8,7 @@ import { getJobsList, getJobsListPaginationAndSort, waitJobs } from '../../../..
 import { flushAndRunMultipleServers } from '../../../../shared/extra-utils/server/servers'
 import { uploadVideo } from '../../../../shared/extra-utils/videos/videos'
 import { dateIsValid } from '../../../../shared/extra-utils/miscs/miscs'
+import { Job } from '../../../../shared/models/server'
 
 const expect = chai.expect
 
@@ -50,7 +51,7 @@ describe('Test jobs', function () {
     if (job.type === 'videos-views') job = res.body.data[1]
 
     expect(job.state).to.equal('completed')
-    expect(job.type).to.equal('activitypub-follow')
+    expect(job.type.startsWith('activitypub-')).to.be.true
     expect(dateIsValid(job.createdAt)).to.be.true
     expect(dateIsValid(job.processedOn)).to.be.true
     expect(dateIsValid(job.finishedOn)).to.be.true
diff --git a/server/tests/api/travis-1.sh b/server/tests/api/travis-1.sh
deleted file mode 100644 (file)
index db4021b..0000000
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env sh
-
-set -eu
-
-checkParamFiles=$(find server/tests/api/check-params -type f | grep -v index.ts | xargs echo)
-notificationsFiles=$(find server/tests/api/notifications -type f | grep -v index.ts | xargs echo)
-searchFiles=$(find server/tests/api/search -type f | grep -v index.ts | xargs echo)
-
-MOCHA_PARALLEL=true mocha --timeout 5000 --exit --require ts-node/register --bail \
-    $notificationsFiles $searchFiles $checkParamFiles
diff --git a/server/tests/api/travis-2.sh b/server/tests/api/travis-2.sh
deleted file mode 100644 (file)
index ba7a061..0000000
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/env sh
-
-set -eu
-
-serverFiles=$(find server/tests/api/server -type f | grep -v index.ts | xargs echo)
-usersFiles=$(find server/tests/api/users -type f | grep -v index.ts | xargs echo)
-
-MOCHA_PARALLEL=true mocha --timeout 5000 --exit --require ts-node/register --bail \
-    $serverFiles $usersFiles
diff --git a/server/tests/api/travis-3.sh b/server/tests/api/travis-3.sh
deleted file mode 100644 (file)
index 8245722..0000000
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/env sh
-
-set -eu
-
-videosFiles=$(find server/tests/api/videos -type f | grep -v index.ts | xargs echo)
-
-MOCHA_PARALLEL=true mocha --timeout 5000 --exit --require ts-node/register --bail \
-    $videosFiles
diff --git a/server/tests/api/travis-4.sh b/server/tests/api/travis-4.sh
deleted file mode 100644 (file)
index 8759861..0000000
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/env sh
-
-set -eu
-
-redundancyFiles=$(find server/tests/api/redundancy -type f | grep -v index.ts | xargs echo)
-activitypubFiles=$(find server/tests/api/activitypub -type f | grep -v index.ts | xargs echo)
-
-MOCHA_PARALLEL=true mocha-parallel-tests --max-parallel $1 --timeout 5000 --exit --require ts-node/register --bail \
-    $redundancyFiles $activitypubFiles
index 64f657780b979f4497ec9de2b4d39696d6f0e001..65a8eafb85e13ab39004e75dcd5497d0da74d98d 100644 (file)
@@ -85,7 +85,7 @@ describe('Test video update scheduler', function () {
   })
 
   it('Should wait some seconds and have the video in public privacy', async function () {
-    this.timeout(20000)
+    this.timeout(50000)
 
     await wait(15000)
     await waitJobs(servers)
index 42250886c2b2fc04f5956003e1dce27a16e5d587..08f41a7e7503ef564fb0d69d35bd9787dc3e3b0e 100644 (file)
@@ -1,10 +1,10 @@
 /* tslint:disable:no-unused-expression */
 
 import * as chai from 'chai'
-import { basename, isAbsolute, join, resolve } from 'path'
+import { basename, dirname, isAbsolute, join, resolve } from 'path'
 import * as request from 'supertest'
 import * as WebTorrent from 'webtorrent'
-import { pathExists, readFile } from 'fs-extra'
+import { ensureDir, pathExists, readFile } from 'fs-extra'
 import * as ffmpeg from 'fluent-ffmpeg'
 
 const expect = chai.expect
@@ -59,12 +59,12 @@ async function testImage (url: string, imageName: string, imagePath: string, ext
   expect(data.length).to.be.below(maxLength)
 }
 
-function buildAbsoluteFixturePath (path: string, customTravisPath = false) {
+function buildAbsoluteFixturePath (path: string, customCIPath = false) {
   if (isAbsolute(path)) {
     return path
   }
 
-  if (customTravisPath && process.env.TRAVIS) return join(process.env.HOME, 'fixtures', path)
+  if (customCIPath && process.env.GITLAB_CI) return join(root(), 'cached-fixtures', path)
 
   return join(root(), 'server', 'tests', 'fixtures', path)
 }
@@ -72,6 +72,8 @@ function buildAbsoluteFixturePath (path: string, customTravisPath = false) {
 async function generateHighBitrateVideo () {
   const tempFixturePath = buildAbsoluteFixturePath('video_high_bitrate_1080p.mp4', true)
 
+  await ensureDir(dirname(tempFixturePath))
+
   const exists = await pathExists(tempFixturePath)
   if (!exists) {
 
index 1961a8762509611ac1255d38f964ba65673f92e2..dfe840d8c9bb1e16db2f8c925c75cef9fc4b428e 100644 (file)
@@ -1,6 +1,5 @@
 import { QueryTypes, Sequelize } from 'sequelize'
 import { ServerInfo } from '../server/servers'
-import { PluginType } from '../../models/plugins/plugin.type'
 
 let sequelizes: { [ id: number ]: Sequelize } = {}
 
@@ -10,7 +9,7 @@ function getSequelize (internalServerNumber: number) {
   const dbname = 'peertube_test' + internalServerNumber
   const username = 'peertube'
   const password = 'peertube'
-  const host = 'localhost'
+  const host = process.env.GITLAB_CI ? 'postgres' : 'localhost'
   const port = 5432
 
   const seq = new Sequelize(dbname, username, password, {
index 692b5e24d4b6f8eb01c3a7627cb1970321b58bb6..11b570f609764d301ad997d5c287789c19bd285a 100644 (file)
@@ -2,6 +2,7 @@ import * as request from 'supertest'
 import { Job, JobState } from '../../models'
 import { wait } from '../miscs/miscs'
 import { ServerInfo } from './servers'
+import { inspect } from 'util'
 
 function getJobsList (url: string, accessToken: string, state: JobState) {
   const path = '/api/v1/jobs/' + state
@@ -49,7 +50,9 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) {
           .then(res => res.body.data)
           .then((jobs: Job[]) => jobs.filter(j => j.type !== 'videos-views'))
           .then(jobs => {
-            if (jobs.length !== 0) pendingRequests = true
+            if (jobs.length !== 0) {
+              pendingRequests = true
+            }
           })
         tasks.push(p)
       }
index 40cf7f0f38c8e3ade6606ad264549a12c1957532..a0720d778ec001ea95ad54fcaa581ba800634e1f 100644 (file)
@@ -79,8 +79,8 @@ function flushTests (serverNumber?: number) {
   return new Promise<void>((res, rej) => {
     const suffix = serverNumber ? ` -- ${serverNumber}` : ''
 
-    return exec('npm run clean:server:test' + suffix, err => {
-      if (err) return rej(err)
+    return exec('npm run clean:server:test' + suffix, (err, _stdout, stderr) => {
+      if (err || stderr) return rej(err || new Error(stderr))
 
       return res()
     })
index 352a1f0d33e15f1c6f79a7e2f2ccbc6456f0ead4..c030757357b3e275129bf6b4e13671091dfe191a 100644 (file)
@@ -10,7 +10,7 @@
     * `NODE_APP_INSTANCE=6 NODE_ENV=test npm run start` and check migrations does not fail
  * Run `rm -r node_modules && rm -r client/node_modules && yarn install --pure-lockfile && npm run build` to see if all the supported languages compile correctly
  * Update https://peertube2.cpy.re and check it works correctly
- * Check Travis tests are green
+ * Check CI tests are green
  * Run E2E tests: `BROWSERSTACK_USER=my_user BROWSERSTACK_KEY=my_key npm run e2e`
  * Release: `GITHUB_TOKEN=my_token npm run release -- 1.x.x`
  * Create a dedicated branch: `git checkout -b release/1.x.x && git push origin release/1.x.x`
index 3d6b1466a434208bcf7c6f4ed0e17ea93e70df0f..894d5e81bc69dfdaa23562003a0eab5eb9bfd2c4 100644 (file)
--- a/yarn.lock
+++ b/yarn.lock
   dependencies:
     "@types/node" "*"
 
-"@types/bluebird@*", "@types/bluebird@3.5.21":
+"@types/bluebird@*", "@types/bluebird@3.5.27":
+  version "3.5.27"
+  resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.27.tgz#61eb4d75dc6bfbce51cf49ee9bbebe941b2cb5d0"
+  integrity sha512-6BmYWSBea18+tSjjSC3QIyV93ZKAeNWGM7R6aYt1ryTZXrlHF+QLV0G2yV0viEGVyRkyQsWfMoJ0k/YghBX5sQ==
+
+"@types/bluebird@3.5.21":
   version "3.5.21"
   resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.21.tgz#567615589cc913e84a28ecf9edb031732bdf2634"
   integrity sha512-6UNEwyw+6SGMC/WMI0ld0PS4st7Qq51qgguFrFizOSpGvZiqe9iswztFSdZvwJBEhLOy2JaxNE6VC7yMAlbfyQ==
@@ -4678,7 +4683,7 @@ mkdirp@0.5.1, mkdirp@0.x.x, mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1:
   dependencies:
     minimist "0.0.8"
 
-mocha-parallel-tests@^2.1.0:
+mocha-parallel-tests@^2.2.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/mocha-parallel-tests/-/mocha-parallel-tests-2.2.1.tgz#616b316c9f20c956a6ce18cd38d45e7db3045b99"
   integrity sha512-9t0NtQ2V/KrNKaUj08RVZp0dqJxeRMTGvC7JA5Daw/LA8GOP+4P7NxRgU0e2sM0JRWYxSakXtpE+MEVGX1ifuw==