Migrations/neko-rooms
1
0
Fork 0
mirror of https://github.com/m1k1o/neko-rooms.git synced 2026-11-02 22:51:34 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9 from m1k1o/persistent-data

Browse Source 
Allow persistent data for container
This commit is contained in:
Miroslav Šedivý
2021-05-22 20:11:35 +02:00
committed by GitHub
parent 98d56f40ef dcdd2ab8a7
commit a024f8f883
13 changed files with 363 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
OpenApi.yml
+18
View File
@@ -243,6 +243,20 @@ components:
format: datetime
example: "2021-03-07T21:56:34Z"
RoomMount:
type: object
properties:
type:
type: string
enum: [ private, template, public ]
example: private
host_path:
type: string
example: /profile
container_path:
type: string
example: /home/neko/.config/chromium
RoomSettings:
type: object
properties:
@@ -291,6 +305,10 @@ components:
type: string
example:
CUSTOM_ENV: custom value
mounts:
type: array
items:
$ref: '#/components/schemas/RoomMount'
RoomStats:
type: object
README.md
+2 -2
View File
@@ -90,8 +90,8 @@ docker-compose up -d
- [x] add GUI
- [x] add HTTPS support
- [x] add authentication provider for traefik
- [x] allow specifying custom ENV variables (TODO: FE.)
- [ ] allow mounting direcotries for persistent data
- [x] allow specifying custom ENV variables
- [x] allow mounting direcotries for persistent data
- [ ] add upgrade button
- [ ] auto pull images, that do not exist
- [ ] add bearer token to for API
client/src/api/.openapi-generator/VERSION
+1 -1
View File
@@ -1 +1 @@
5.1.1-SNAPSHOT
5.2.0-SNAPSHOT
client/src/api/api.ts
+42
View File
@@ -107,6 +107,42 @@ export interface RoomMember {
*/
muted?: boolean;
}
/**
*
* @export
* @interface RoomMount
*/
export interface RoomMount {
/**
*
* @type {string}
* @memberof RoomMount
*/
type?: RoomMountTypeEnum;
/**
*
* @type {string}
* @memberof RoomMount
*/
host_path?: string;
/**
*
* @type {string}
* @memberof RoomMount
*/
container_path?: string;
}
/**
* @export
* @enum {string}
*/
export enum RoomMountTypeEnum {
private = 'private',
template = 'template',
public = 'public'
}
/**
*
* @export
@@ -203,6 +239,12 @@ export interface RoomSettings {
* @memberof RoomSettings
*/
envs?: { [key: string]: string; };
/**
*
* @type {Array<RoomMount>}
* @memberof RoomSettings
*/
mounts?: Array<RoomMount>;
}
/**
*
client/src/components/RoomInfo.vue
+9
View File
@@ -145,6 +145,15 @@
</tbody>
</template>
</v-simple-table>
<div class="my-3 headline">Mounts</div>
<v-simple-table>
<template v-slot:default>
<tbody>
<tr v-for="({ host_path, container_path }, index) in settings.mounts" :key="index"><td style="width:50%;">{{ host_path }}</td><td>{{ container_path }}</td></tr>
</tbody>
</template>
</v-simple-table>
</template>
</div>
</template>
client/src/components/RoomsCreate.vue
+75 -5
View File
@@ -14,7 +14,7 @@
<v-text-field
label="Name"
v-model="data.name"
:rules="[ rules.slug ]"
:rules="[ rules.minLen(2), rules.containerNameStart, rules.containerName ]"
autocomplete="off"
></v-text-field>
</v-col>
@@ -187,7 +187,7 @@
<v-btn @click="addEnv" icon color="green"><v-icon>mdi-plus</v-icon></v-btn>
</v-row>
<v-row align="center" v-for="({ key, val }, index) in envList" :key="index">
<v-col class="pb-0">
<v-col class="py-0">
<v-text-field
label="Key"
:value="key"
@@ -195,7 +195,8 @@
autocomplete="off"
></v-text-field>
</v-col>
<v-col class="pb-0">
<div> : </div>
<v-col class="py-0">
<v-text-field
label="Value"
:value="val"
@@ -207,6 +208,49 @@
<v-btn @click="delEnv(index)" icon color="red"><v-icon>mdi-close</v-icon></v-btn>
</div>
</v-row>
<v-row align="center" no-gutters class="mt-3">
<h2> Mounts </h2>
<v-btn @click="data.mounts = [ ...data.mounts, { type: 'private', host_path: '', container_path: '' }]" icon color="green"><v-icon>mdi-plus</v-icon></v-btn>
</v-row>
<v-row align="center" class="mb-2" v-for="({ type, host_path, container_path }, index) in data.mounts" :key="index">
<v-col class="py-0" cols="2">
<v-select
label="Type"
:items="mountTypes"
:value="type"
@input="$set(data.mounts, index, { type: $event, host_path, container_path })"
></v-select>
</v-col>
<v-col class="py-0 pl-0">
<v-text-field
label="Host path"
:value="host_path"
@input="$set(data.mounts, index, { type, host_path: $event, container_path })"
:rules="[ rules.absolutePath ]"
autocomplete="off"
></v-text-field>
</v-col>
<div> : </div>
<v-col class="py-0">
<v-text-field
label="Container path"
:value="container_path"
@input="$set(data.mounts, index, { type, host_path, container_path: $event})"
:rules="[ rules.absolutePath ]"
autocomplete="off"
></v-text-field>
</v-col>
<div>
<v-btn @click="$delete(data.mounts, index)" icon color="red"><v-icon>mdi-close</v-icon></v-btn>
</div>
</v-row>
<v-row align="center" no-gutters v-if="data.mounts.length > 0">
<p>
<strong>Private</strong>: Host path is relative to <code class="mx-1">&lt;storage path&gt;/rooms/&lt;room name&gt;/</code>. <br />
<strong>Template</strong>: Host path is relative to <code class="mx-1">&lt;storage path&gt;/templates/</code> and is readonly. <br />
<strong>Public</strong>: Host path must be whitelisted in config and exists on the host.
</p>
</v-row>
</template>
</v-form>
</v-card-text>
@@ -272,15 +316,24 @@ export default class RoomsCreate extends Vue {
required(val: any) {
return val === null || typeof val === 'undefined' || val === "" ? 'This filed is mandatory.' : true
},
minLen: (min: number) =>
(val: string) =>
val ? (val.length >= min || 'This field must have atleast ' + min + ' characters') : true,
onlyPositive(val: number) {
return val < 0 ? 'Value cannot be negative.' : true
},
nonZero(val: string) {
return val === "0" ? 'Value cannot be zero.' : true
},
slug(val: string) {
return val && !/^[A-Za-z0-9-_.]+$/.test(val) ? 'Should only contain A-Z a-z 0-9 - _ .' : true
containerName(val: string) {
return val && !/^[a-zA-Z0-9_.-]+$/.test(val) ? 'Must only contain a-z A-Z 0-9 _ . -' : true
},
containerNameStart(val: string) {
return val && /^[_.-]/.test(val) ? 'Cannot start with _ . -' : true
},
absolutePath(val: string) {
return val[0] !== "/" ? 'Must be absolute path, starting with /.' : true
}
}
get nekoImages() {
@@ -299,6 +352,23 @@ export default class RoomsCreate extends Vue {
return this.$store.state.availableScreens
}
get mountTypes() {
return [
{
text: 'Private',
value: 'private',
},
{
text: 'Template',
value: 'template',
},
{
text: 'Public',
value: 'public',
},
]
}
addEnv() {
Vue.set(this, 'envList', [ ...this.envList, { key: '', val: '' } ])
}
client/src/store/state.ts
+5
View File
@@ -37,6 +37,11 @@ export const state = {
// eslint-disable-next-line
broadcast_pipeline: '',
// eslint-disable-next-line
envs: {},
// eslint-disable-next-line
mounts: [],
} as RoomSettings,
videoCodecs: [
"VP8",
dev/.gitignore
+2
View File
@@ -0,0 +1,2 @@
data
ext
dev/start
+11 -1
View File
@@ -9,12 +9,22 @@ fi
export $(cat ../.env | sed 's/#.*//g' | xargs)
DATA_PATH="./data"
mkdir -p "${DATA_PATH}"
EXTERNAL_PATH="./ext"
mkdir -p "${EXTERNAL_PATH}"
docker run --rm -it \
--name="neko_rooms_server" \
-v "${PWD}/../:/app" \
-v "`realpath ..`:/app" \
-v "`realpath ${DATA_PATH}`:/data" \
-e "TZ=${TZ}" \
-e "NEKO_ROOMS_EPR=${NEKO_ROOMS_EPR}" \
-e "NEKO_ROOMS_NAT1TO1=${NEKO_ROOMS_NAT1TO1}" \
-e "NEKO_ROOMS_STORAGE_INTERNAL=/data" \
-e "NEKO_ROOMS_STORAGE_EXTERNAL=`realpath ${DATA_PATH}`" \
-e "NEKO_ROOMS_MOUNTS_WHITELIST=`realpath ${EXTERNAL_PATH}`" \
-e "NEKO_ROOMS_TRAEFIK_DOMAIN=${NEKO_ROOMS_TRAEFIK_DOMAIN}" \
-e "NEKO_ROOMS_TRAEFIK_ENTRYPOINT=${NEKO_ROOMS_TRAEFIK_ENTRYPOINT}" \
-e "NEKO_ROOMS_TRAEFIK_NETWORK=${NEKO_ROOMS_TRAEFIK_NETWORK}" \
internal/config/room.go
+59
View File
@@ -1,9 +1,11 @@
package config
import (
"path/filepath"
"strconv"
"strings"
dockerNames "github.com/docker/docker/daemon/names"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
"github.com/spf13/viper"
@@ -16,6 +18,12 @@ type Room struct {
NAT1To1IPs []string
NekoImages []string
StorageEnabled bool
StorageInternal string
StorageExternal string
MountsWhitelist []string
InstanceName string
InstanceUrl string
@@ -50,6 +58,28 @@ func (Room) Init(cmd *cobra.Command) error {
return err
}
// Data
cmd.PersistentFlags().Bool("storage.enabled", true, "whether storage is enabled, where peristent containers data will be stored")
if err := viper.BindPFlag("storage.enabled", cmd.PersistentFlags().Lookup("storage.enabled")); err != nil {
return err
}
cmd.PersistentFlags().String("storage.external", "", "external absolute path (on the host) to storage folder")
if err := viper.BindPFlag("storage.external", cmd.PersistentFlags().Lookup("storage.external")); err != nil {
return err
}
cmd.PersistentFlags().String("storage.internal", "/data", "internal absolute path (inside container) to storage folder")
if err := viper.BindPFlag("storage.internal", cmd.PersistentFlags().Lookup("storage.internal")); err != nil {
return err
}
cmd.PersistentFlags().StringSlice("mounts.whitelist", []string{}, "whitelisted public mounts for containers")
if err := viper.BindPFlag("mounts.whitelist", cmd.PersistentFlags().Lookup("mounts.whitelist")); err != nil {
return err
}
// Instance
cmd.PersistentFlags().String("instance.name", "neko-rooms", "unique instance name (if running muliple on the same host)")
@@ -120,7 +150,36 @@ func (s *Room) Set() {
s.NAT1To1IPs = viper.GetStringSlice("nat1to1")
s.NekoImages = viper.GetStringSlice("neko_images")
s.StorageEnabled = viper.GetBool("storage.enabled")
s.StorageInternal = viper.GetString("storage.internal")
s.StorageExternal = viper.GetString("storage.external")
if s.StorageInternal != "" && s.StorageExternal != "" {
s.StorageInternal = filepath.Clean(s.StorageInternal)
s.StorageExternal = filepath.Clean(s.StorageExternal)
if !filepath.IsAbs(s.StorageInternal) || !filepath.IsAbs(s.StorageExternal) {
log.Panic().Msg("invalid `storage.internal` or `storage.external`, must be an absolute path")
}
} else {
log.Warn().Msg("missing `storage.internal` or `storage.external`, storage is unavailable")
s.StorageEnabled = false
}
s.MountsWhitelist = viper.GetStringSlice("mounts.whitelist")
for _, path := range s.MountsWhitelist {
path = filepath.Clean(path)
if !filepath.IsAbs(path) {
log.Panic().Msg("invalid `mounts.whitelist`, must be an absolute path")
}
}
s.InstanceName = viper.GetString("instance.name")
if !dockerNames.RestrictedNamePattern.MatchString(s.InstanceName) {
log.Panic().Msg("invalid `instance.name`, must match " + dockerNames.RestrictedNameChars)
}
s.InstanceUrl = viper.GetString("instance.url")
s.TraefikDomain = viper.GetString("traefik.domain")
internal/room/manager.go
+108 -1
View File
@@ -4,13 +4,18 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"path"
"path/filepath"
"strings"
dockerTypes "github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
dockerMount "github.com/docker/docker/api/types/mount"
network "github.com/docker/docker/api/types/network"
"github.com/docker/docker/api/types/strslice"
dockerClient "github.com/docker/docker/client"
dockerNames "github.com/docker/docker/daemon/names"
"github.com/docker/go-connections/nat"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
@@ -21,7 +26,11 @@ import (
)
const (
frontendPort = 8080
frontendPort = 8080
templateStoragePath = "./templates"
privateStoragePath = "./rooms"
privateStorageUid = 1000
privateStorageGid = 1000
)
func New(config *config.Room) *RoomManagerCtx {
@@ -83,6 +92,14 @@ func (manager *RoomManagerCtx) FindByName(name string) (*types.RoomEntry, error)
}
func (manager *RoomManagerCtx) Create(settings types.RoomSettings) (string, error) {
if settings.Name != "" && !dockerNames.RestrictedNamePattern.MatchString(settings.Name) {
return "", fmt.Errorf("invalid container name, must match " + dockerNames.RestrictedNameChars)
}
if !manager.config.StorageEnabled && len(settings.Mounts) > 0 {
return "", fmt.Errorf("mounts cannot be specified, because storage is disabled or unavailable")
}
if in, _ := utils.ArrayIn(settings.NekoImage, manager.config.NekoImages); !in {
return "", fmt.Errorf("invalid neko image")
}
@@ -177,6 +194,70 @@ func (manager *RoomManagerCtx) Create(settings types.RoomSettings) (string, erro
env = append(env, fmt.Sprintf("NEKO_NAT1TO1=%s", strings.Join(manager.config.NAT1To1IPs, ",")))
}
mounts := []dockerMount.Mount{}
for _, mount := range settings.Mounts {
readOnly := false
hostPath := filepath.Clean(mount.HostPath)
containerPath := filepath.Clean(mount.ContainerPath)
if !filepath.IsAbs(hostPath) || !filepath.IsAbs(containerPath) {
return "", fmt.Errorf("mount paths must be absolute")
}
// private container's data
if mount.Type == types.MountPrivate {
// ensure that target exists
internalPath := path.Join(manager.config.StorageInternal, privateStoragePath, roomName, hostPath)
if err := os.MkdirAll(internalPath, os.ModePerm); err != nil {
return "", err
}
if err := utils.ChownR(internalPath, privateStorageUid, privateStorageGid); err != nil {
return "", err
}
// prefix host path
hostPath = path.Join(manager.config.StorageExternal, privateStoragePath, roomName, hostPath)
} else if mount.Type == types.MountTemplate {
// readonly template data
readOnly = true
// prefix host path
hostPath = path.Join(manager.config.StorageExternal, templateStoragePath, hostPath)
} else if mount.Type == types.MountPublic {
// public whitelisted mounts
var isAllowed = false
for _, path := range manager.config.MountsWhitelist {
if strings.HasPrefix(hostPath, path) {
isAllowed = true
break
}
}
if !isAllowed {
return "", fmt.Errorf("mount path is not whitelisted in config")
}
} else {
return "", fmt.Errorf("unknown mount type %q", mount.Type)
}
mounts = append(mounts,
dockerMount.Mount{
Type: dockerMount.TypeBind,
Source: hostPath,
Target: containerPath,
ReadOnly: readOnly,
Consistency: dockerMount.ConsistencyDefault,
BindOptions: &dockerMount.BindOptions{
Propagation: dockerMount.PropagationRPrivate,
NonRecursive: false,
},
},
)
}
config := &container.Config{
// Hostname
Hostname: containerName,
@@ -211,6 +292,8 @@ func (manager *RoomManagerCtx) Create(settings types.RoomSettings) (string, erro
},
// Total shm memory usage
ShmSize: 2 * 10e9,
// Mounts specs used by the container
Mounts: mounts,
}
networkingConfig := &network.NetworkingConfig{
@@ -295,10 +378,34 @@ func (manager *RoomManagerCtx) GetSettings(id string) (*types.RoomSettings, erro
return nil, err
}
privateStorageRoot := path.Join(manager.config.StorageExternal, privateStoragePath, roomName)
templateStorageRoot := path.Join(manager.config.StorageExternal, templateStoragePath)
mounts := []types.RoomMount{}
for _, mount := range container.Mounts {
mountType := types.MountPublic
hostPath := mount.Source
if strings.HasPrefix(hostPath, privateStorageRoot) {
mountType = types.MountPrivate
hostPath = strings.TrimPrefix(hostPath, privateStorageRoot)
} else if strings.HasPrefix(hostPath, templateStorageRoot) {
mountType = types.MountTemplate
hostPath = strings.TrimPrefix(hostPath, templateStorageRoot)
}
mounts = append(mounts, types.RoomMount{
Type: mountType,
HostPath: hostPath,
ContainerPath: mount.Destination,
})
}
settings := types.RoomSettings{
Name: roomName,
NekoImage: nekoImage,
MaxConnections: epr.Max - epr.Min + 1,
Mounts: mounts,
}
err = settings.FromEnv(container.Config.Env)
internal/types/room.go
+16 -1
View File
@@ -39,6 +39,20 @@ type RoomEntry struct {
Created time.Time `json:"created"`
}
type MountType string
const (
MountPrivate MountType = "private"
MountTemplate MountType = "template"
MountPublic MountType = "public"
)
type RoomMount struct {
Type MountType `json:"type"`
HostPath string `json:"host_path"`
ContainerPath string `json:"container_path"`
}
type RoomSettings struct {
Name string `json:"name"`
NekoImage string `json:"neko_image"`
@@ -59,7 +73,8 @@ type RoomSettings struct {
BroadcastPipeline string `json:"broadcast_pipeline,omitempty"`
Envs map[string]string `json:"envs"`
Envs map[string]string `json:"envs"`
Mounts []RoomMount `json:"mounts"`
}
func (settings *RoomSettings) ToEnv() []string {
internal/utils/fs.go
+15
View File
@@ -0,0 +1,15 @@
package utils
import (
"os"
"path/filepath"
)
func ChownR(path string, uid, gid int) error {
return filepath.Walk(path, func(name string, info os.FileInfo, err error) error {
if err == nil {
err = os.Chown(name, uid, gid)
}
return err
})
}