optimization many

Author: leffss

Dockerfile

@@ -4,7 +4,7 @@ COPY ./sources.list /etc/apt
 ADD . /devops
 WORKDIR /devops
 # RUN apt-get update && apt-get install python3-dev default-libmysqlclient-dev sshpass -y
-RUN cd /devops && pip install -i https://pypi.douban.com/simple -r requirements.txt
+RUN cd /devops && pip install -i https://mirrors.aliyun.com/pypi/simple -r requirements.txt
 RUN cd /devops && echo_supervisord_conf > /etc/supervisord.conf && cat deamon.ini >> /etc/supervisord.conf && \
 	sed -i 's/nodaemon=false/nodaemon=true/g' /etc/supervisord.conf
 RUN dpkg -i sshpass_1.06-1_amd64.deb

README.md

@@ -1,6 +1,6 @@
 # devops
 基于 python 3.7 + django 2.2.3 + channels 2.2.0 + celery 4.3.0 + ansible 2.8.5 + AdminLTE-3.0.0 实现的运维 devops 管理系统。具体见 `screenshots` 文件夹中的效果预览图。
-本人为运维工程师，非专业开发，代码写得烂，不喜勿喷，欢迎指教。功能持续完善中。
+本人为运维工程师，非专业开发，此项目各个功能模块都是现学现用，可能有点地方没有考虑到合理和性能的问题，代码写得烂，不喜勿喷，欢迎 issue。功能持续完善中。
 
 
 # 部署安装
@@ -26,13 +26,14 @@ docker run --name redis-server -p 6379:6379 -d redis:latest
 docker run --name guacd -p 4822:4822 -d guacamole/guacd
 ```
 - rdp 与 vnc 连接支持所需，非必须
+- windows rdp 必须设置为`允许运行任意版本远程桌面的计算机连接(较不安全)(L)`才能连接
 
 **安装 python 依赖库**
 ```bash
 # 安装相关库
-pip3 install -i https://pypi.douban.com/simple -r requirements.txt
+pip3 install -i https://mirrors.aliyun.com/pypi/simple -r requirements.txt
 ```
-- -i 指定豆瓣源，国外源慢得一逼，我大天朝威武
+- -i 指定阿里源，国外源慢得一逼，我大天朝威武
 
 **修改 devops/settings.py 配置**
 
@@ -43,14 +44,17 @@ pip3 install -i https://pypi.douban.com/simple -r requirements.txt
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.mysql',
+        # 'ENGINE': 'db_pool.mysql',     # db_pool.mysql 为重写 django 官方 mysql 连接库实现了真正的连接池
         'NAME': 'devops',
         'USER':'devops',
         'PASSWORD':'devops',
         'HOST':'127.0.0.1',
         'PORT':'3306',
+        'CONN_MAX_AGE': 600,    # 如果使用 db_pool.mysql 尽量不要设置此参数
+        # 数据库连接池大小，mysql 总连接数大小为：连接池大小 * 服务进程数
+        'DB_POOL_SIZE': 20,     # 默认 5 个
         'OPTIONS': {
             'init_command': "SET sql_mode='STRICT_TRANS_TABLES'",
-            # 'init_command': "SET sql_mode=''",
          },
     }
 }
@@ -81,12 +85,14 @@ export PYTHONOPTIMIZE=1		# 解决 celery 不允许创建子进程的问题
 nohup gunicorn -c gunicorn.cfg devops.wsgi:application > logs/gunicorn.log 2>&1 &
 nohup daphne -b 0.0.0.0 -p 8001 --access-log=logs/daphne_access.log devops.asgi:application > logs/daphne.log 2>&1 &
 nohup python3 manage.py sshd > logs/sshd.log 2>&1 &
-nohup celery -A devops worker -l info -c 3 --max-tasks-per-child 40 --prefetch-multiplier 1 > logs/celery.log 2>&1 &
+nohup celery -A devops worker -l info -c 3 --max-tasks-per-child 40 --prefetch-multiplier 1 --pidfile logs/celery_worker.pid > logs/celery.log 2>&1 &
+nohup celery -A devops beat -l info --pidfile logs/celery_worker.pid > logs/celery_beat.log 2>&1 &
 ```
 - gunicorn  处理 http 请求，监听 8000 端口
 - daphne 处理 websocket 请求，监听 8001 端口
 - sshd 为 ssh 代理服务器，监听 2222 端口，提供调用 securecrt、xshell、putty 以及 winscp 客户端支持，非必须
 - celery 后台任务处理，`export PYTHONOPTIMIZE=1` 此环境变量非常重要，不设置无法后台运行 ansible api
+- celery_beat 处理 `devops/settings.py` 中设置的 `CELERY_BEAT_SCHEDULE` 定时任务
 
 **nginx  前端代理**
 ```
@@ -244,6 +250,21 @@ systemctl start nginx
 
 # 升级日志
 
+### ver1.8.6
+优化执行 playbook 逻辑：允许指定组；
+
+优化终端登陆后 su 跳转逻辑：由管理员能够跳转变更为有权限就可跳转；
+
+~~新增 apscheduler 在启动时执行清空 TerminalSession 表；~~
+
+新增 django mysql 连接 ENGINE 优化版本：真正的支持连接池；
+
+修正 clissh 使用 Zmodem 使用 sz 和 rz 时的 BUG；
+
+更新 xterm.js 到 v3.14.5 版本，支持 webssh 与 webtelnet 复制文本内容；
+
+- 此版本新增了一个任务调度模块，目前还是一个不完善的功能，不要太在意，仅供参考
+
 ### ver1.8.5
 新增批量操作，比如批量删除，批量更新等；
 
@@ -294,7 +315,7 @@ systemctl start nginx
 优化 UI 界面，加入动态效果；
 
 ### ver1.7.5
-修正强制断开 clissh 后保存 2 次终端日志的BUG；
+修正强制断开 clissh 后保存 2 次终端日志的 BUG；
 
 新增会话在一定时间内无操作自动断开功能（默认 30 分钟，settings.py 中可配置）；
 
@@ -384,6 +405,11 @@ linux 平台下使用 celery 任务保存终端会话日志与录像（windows 
 - [ ] docker 容器管理
 - [ ] k8s 集群管理
 - [ ] 自动化部署CI/CD
+- [ ] webssh 与 webtelnet 的 zmodem(sz, rz) 支持
 
+# MIT License
+```
+Copyright (c) 2019-2020 leffss
+```
 
 更多新功能不断探索发现中.

apps/batch/__pycache__/__init__.cpython-37.pyc

@@ -1,6 +1,7 @@
 from channels.generic.websocket import WebsocketConsumer
 from django.conf import settings
 from server.models import RemoteUserBindHost
+from user.models import User
 from django.db.models import Q
 from asgiref.sync import async_to_sync
 from util.tool import gen_rand_char
@@ -20,7 +21,7 @@
 
 
 def get_hosts(issuperuser, ids, username):
-    if issuperuser:
+    if issuperuser and username == 'admin':
         return RemoteUserBindHost.objects.filter(
                         Q(enabled=True),
                         Q(id__in=ids.split(',')),
@@ -138,7 +139,7 @@ def receive(self, text_data=None, bytes_data=None):
                     user=self.session.get('username'),
                     user_agent=self.user_agent,
                     client=self.client,
-                    issuperuser=self.session.get('issuperuser', False),
+                    issuperuser=True if '登陆后su跳转超级用户' in self.session[settings.INIT_PERMISSION]['titles'] else False,
                 )  # 执行
 
             else:
@@ -270,7 +271,7 @@ def receive(self, text_data=None, bytes_data=None):
                     user=self.session.get('username'),
                     user_agent=self.user_agent,
                     client=self.client,
-                    issuperuser=self.session.get('issuperuser', False),
+                    issuperuser=True if '登陆后su跳转超级用户' in self.session[settings.INIT_PERMISSION]['titles'] else False,
                 )  # 执行
 
             else:
@@ -410,7 +411,7 @@ def receive(self, text_data=None, bytes_data=None):
                     user=self.session.get('username'),
                     user_agent=self.user_agent,
                     client=self.client,
-                    issuperuser=self.session.get('issuperuser', False),
+                    issuperuser=True if '登陆后su跳转超级用户' in self.session[settings.INIT_PERMISSION]['titles'] else False,
                 )  # 执行
             else:
                 self.message['status'] = 1
@@ -529,6 +530,8 @@ def receive(self, text_data=None, bytes_data=None):
                     hostinfo['port'] = host.port
                     hostinfo['username'] = host.remote_user.username
                     hostinfo['password'] = host.remote_user.password
+                    user = User.objects.get(id=int(self.session.get('userid')))
+                    hostinfo['groups'] = [x.group_name for x in host.host_group.filter(user=user)]
                     if host.remote_user.enabled:
                         hostinfo['superusername'] = host.remote_user.superusername
                         hostinfo['superpassword'] = host.remote_user.superpassword
@@ -541,7 +544,7 @@ def receive(self, text_data=None, bytes_data=None):
                     user=self.session.get('username'),
                     user_agent=self.user_agent,
                     client=self.client,
-                    issuperuser=self.session.get('issuperuser', False),
+                    issuperuser=True if '登陆后su跳转超级用户' in self.session[settings.INIT_PERMISSION]['titles'] else False,
                 )  # 执行
             else:
                 self.message['status'] = 1
@@ -672,7 +675,7 @@ def receive(self, text_data=None, bytes_data=None):
                     user=self.session.get('username'),
                     user_agent=self.user_agent,
                     client=self.client,
-                    issuperuser=self.session.get('issuperuser', False),
+                    issuperuser=True if '登陆后su跳转超级用户' in self.session[settings.INIT_PERMISSION]['titles'] else False,
                 )  # 执行
             else:
                 self.message['status'] = 1

apps/batch/__pycache__/models.cpython-37.pyc

@@ -0,0 +1,16 @@
+from os import path
+from django.apps import AppConfig
+
+VERBOSE_APP_NAME = "scheduler"
+
+
+def get_current_app_name(file):
+    return path.dirname(file).replace('\\', '/').split('/')[-1]
+
+
+class AppVerboseNameConfig(AppConfig):
+    name = get_current_app_name(__file__)
+    verbose_name = '任务调度'
+
+
+default_app_config = get_current_app_name(__file__) + '.__init__.AppVerboseNameConfig'

apps/batch/websocket_layer.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

apps/scheduler/__init__.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class SchedulerConfig(AppConfig):
+    name = 'scheduler'

apps/scheduler/admin.py

@@ -0,0 +1,33 @@
+from django.db import models
+
+# Create your models here.
+
+
+class SchedulerHost(models.Model):
+    PROTOCOL_CHOICES = (
+        (1, 'http'),
+        (2, 'https'),
+    )
+    hostname = models.CharField(max_length=128, blank=True, null=True, verbose_name='主机名')
+    ip = models.GenericIPAddressField(verbose_name='主机IP')
+    protocol = models.SmallIntegerField(default=2, choices=PROTOCOL_CHOICES, verbose_name='协议')
+    port = models.SmallIntegerField(default=443, verbose_name='端口')
+    token = models.CharField(max_length=512, verbose_name='token')
+    status = models.BooleanField(default=False, verbose_name='状态')
+    cron = models.IntegerField(default=0, verbose_name='cron任务数')
+    interval = models.IntegerField(default=0, verbose_name='interval任务数')
+    date = models.IntegerField(default=0, verbose_name='date任务数')
+    executed = models.IntegerField(default=0, verbose_name='总共执行任务数')
+    failed = models.IntegerField(default=0, verbose_name='总共失败任务数')
+    memo = models.TextField(blank=True, null=True, verbose_name='备注')
+    update_time = models.DateTimeField('更新时间', blank=True, null=True, auto_now=True)
+    create_time = models.DateTimeField('添加时间', auto_now_add=True)
+
+    def __str__(self):
+        return self.hostname
+
+    class Meta:
+        ordering = ["-create_time"]
+        verbose_name = '调度主机'
+        verbose_name_plural = '调度主机'
+        unique_together = ('ip', 'port')

apps/scheduler/apps.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

apps/scheduler/migrations/__init__.py

@@ -0,0 +1,10 @@
+from django.urls import path
+from . import views
+
+
+app_name = "scheduler"
+urlpatterns = [
+    path('hosts/', views.hosts, name='hosts'),
+    path('host/<int:host_id>/', views.host, name='host'),
+    path('host/<int:host_id>/jobs/', views.jobs, name='jobs'),
+]

apps/scheduler/models.py

@@ -0,0 +1,8 @@
+from django.urls import path
+from . import views_api
+
+
+app_name = "scheduler"
+urlpatterns = [
+    path('client/upload/', views_api.client_upload, name='client_upload'),
+]

apps/scheduler/tests.py

@@ -0,0 +1,72 @@
+from django.shortcuts import render, get_object_or_404
+from util.tool import login_required
+from .models import SchedulerHost
+from django.db.models import Q
+from ratelimit.decorators import ratelimit      # 限速
+from ratelimit import ALL
+from util.rate import rate, key
+import requests
+import urllib3
+import json
+import traceback
+urllib3.disable_warnings()
+# Create your views here.
+
+
+@ratelimit(key=key, rate=rate, method=ALL, block=True)
+@login_required
+def hosts(request):
+    hosts = SchedulerHost.objects.all()
+    return render(request, 'scheduler/hosts.html', locals())
+
+
+@ratelimit(key=key, rate=rate, method=ALL, block=True)
+@login_required
+def host(request, host_id):
+    host = get_object_or_404(
+        SchedulerHost,
+        pk=host_id,
+    )
+    return render(request, 'scheduler/host.html', locals())
+
+
+@ratelimit(key=key, rate=rate, method=ALL, block=True)
+@login_required
+def jobs(request, host_id):
+    host = get_object_or_404(
+        SchedulerHost,
+        pk=host_id,
+    )
+    retry = 2
+    timeout = 5
+    attempts = 0
+    success = False
+    jobs = []
+    while attempts <= retry and not success:
+        try:
+            headers = dict()
+            headers['AUTHORIZATION'] = host.token
+            headers['user-agent'] = 'requests/devops'
+            url = '{protocol}://{ip}:{port}{url}'.format(
+                protocol=host.get_protocol_display(), ip=host.ip, port=host.port, url='/api/job/lists'
+            )
+            res = requests.get(
+                url=url,
+                headers=headers,
+                timeout=timeout,
+                allow_redirects=True,
+                verify=False,
+            )
+            if res.status_code == 200:
+                r = json.loads(res.text)
+                if r['status'] == 0:
+                    jobs = r['data']
+                    success = True
+                else:
+                    attempts += 1
+            else:
+                attempts += 1
+        except Exception as e:
+            print(traceback.format_exc())
+            attempts += 1
+    return render(request, 'scheduler/jobs.html', {'host': host, 'jobs': jobs})