Как исправить Rollback failed в CI/CD

 1# .github/workflows/deploy-with-rollback.yml
 2name: Deploy with Auto Rollback
 3on: [push]
 4
 5jobs:
 6  deploy:
 7    runs-on: ubuntu-latest
 8    steps:
 9    - uses: actions/checkout@v3
10    
11    - name: Create backup
12      run: |
13        # Создание резервной копии
14        kubectl get deployment my-app -o yaml > backup.yaml
15        echo "BACKUP_CREATED=true" >> $GITHUB_ENV
16    
17    - name: Deploy new version
18      run: |
19        kubectl set image deployment/my-app my-app=new-image:latest
20        kubectl rollout status deployment/my-app --timeout=300s
21    
22    - name: Health check
23      run: |
24        # Проверка здоровья после деплоя
25        for i in {1..10}; do
26          if curl -f http://myapp.com/health/; then
27            echo "Deployment successful"
28            exit 0
29          fi
30          sleep 30
31        done
32        echo "Deployment failed, triggering rollback"
33        exit 1
34    
35    - name: Rollback on failure
36      if: failure() && env.BACKUP_CREATED == 'true'
37      run: |
38        echo "Rolling back to previous version..."
39        kubectl apply -f backup.yaml
40        kubectl rollout status deployment/my-app --timeout=300s

 1# rollback_migrations.py
 2import os
 3import django
 4from django.core.management import execute_from_command_line
 5
 6def rollback_migrations(app_name, target_migration):
 7    """Откат миграций к указанной версии"""
 8    try:
 9        # Получаем список примененных миграций
10        from django.db import connection
11        with connection.cursor() as cursor:
12            cursor.execute("""
13                SELECT name FROM django_migrations 
14                WHERE app = %s ORDER BY id DESC
15            """, [app_name])
16            applied_migrations = [row[0] for row in cursor.fetchall()]
17        
18        # Откатываем миграции до целевой
19        for migration in applied_migrations:
20            if migration == target_migration:
21                break
22            execute_from_command_line(['manage.py', 'migrate', app_name, migration])
23            print(f"Rolled back {migration}")
24        
25        print("Migration rollback completed successfully")
26    except Exception as e:
27        print(f"Rollback failed: {e}")
28        raise

 1#!/bin/bash
 2# config-rollback.sh
 3
 4CONFIG_BACKUP_DIR="/backups/config"
 5CURRENT_CONFIG="/etc/myapp/config.yaml"
 6
 7echo "Starting configuration rollback..."
 8
 9# Проверка наличия резервной копии
10if [ ! -f "$CONFIG_BACKUP_DIR/config_backup.yaml" ]; then
11    echo "No backup found, cannot rollback"
12    exit 1
13fi
14
15# Создание резервной копии текущей конфигурации
16cp "$CURRENT_CONFIG" "$CONFIG_BACKUP_DIR/config_current.yaml"
17
18# Восстановление предыдущей конфигурации
19cp "$CONFIG_BACKUP_DIR/config_backup.yaml" "$CURRENT_CONFIG"
20
21# Перезапуск сервиса
22systemctl restart myapp
23
24# Проверка успешности отката
25if systemctl is-active --quiet myapp; then
26    echo "Configuration rollback successful"
27else
28    echo "Configuration rollback failed"
29    # Восстановление текущей конфигурации
30    cp "$CONFIG_BACKUP_DIR/config_current.yaml" "$CURRENT_CONFIG"
31    systemctl restart myapp
32    exit 1
33fi

 1# docker-rollback.yml
 2version: '3.8'
 3services:
 4  myapp:
 5    image: myapp:${IMAGE_TAG:-latest}
 6    ports:
 7      - "8000:8000"
 8    environment:
 9      - DATABASE_URL=${DATABASE_URL}
10    deploy:
11      replicas: 3
12      rollback_config:
13        parallelism: 1
14        delay: 10s
15        failure_action: pause
16        monitor: 60s
17        max_failure_ratio: 0.3
18      update_config:
19        parallelism: 1
20        delay: 10s
21        failure_action: rollback
22        monitor: 60s
23        max_failure_ratio: 0.3

 1# rollback_monitor.py
 2import time
 3import requests
 4from datetime import datetime
 5
 6class RollbackMonitor:
 7    def __init__(self, service_url, health_endpoint="/health/"):
 8        self.service_url = service_url
 9        self.health_endpoint = health_endpoint
10        self.rollback_count = 0
11    
12    def check_service_health(self):
13        """Проверка здоровья сервиса"""
14        try:
15            response = requests.get(f"{self.service_url}{self.health_endpoint}", timeout=10)
16            return response.status_code == 200
17        except Exception as e:
18            print(f"Health check failed: {e}")
19            return False
20    
21    def trigger_rollback(self):
22        """Триггер отката при проблемах"""
23        self.rollback_count += 1
24        print(f"Triggering rollback #{self.rollback_count} at {datetime.now()}")
25        
26        # Логика отката
27        # ... выполнение отката
28        
29        # Проверка после отката
30        time.sleep(30)
31        if self.check_service_health():
32            print("Rollback successful")
33            return True
34        else:
35            print("Rollback failed")
36            return False
37    
38    def monitor_and_rollback(self, max_rollbacks=3):
39        """Мониторинг и автоматический откат"""
40        consecutive_failures = 0
41        
42        while True:
43            if not self.check_service_health():
44                consecutive_failures += 1
45                print(f"Service unhealthy, failure #{consecutive_failures}")
46                
47                if consecutive_failures >= 3 and self.rollback_count < max_rollbacks:
48                    if self.trigger_rollback():
49                        consecutive_failures = 0
50                    else:
51                        print("Max rollbacks reached, manual intervention required")
52                        break
53            else:
54                consecutive_failures = 0
55            
56            time.sleep(60)