cron is a job scheduler on Unix-like operating system. (Wikipedia)

conda environment activating issue

I had an issue with activating the conda environment on a cron job. The problem was that cron runs in a minimal environment without .bashrc or conda init. cron runs /bin/sh, which doesn’t log in or isn’t interactive, and that causes it not to have a PATH to conda’s bin. Furthermore, if we use a system crontab such as /etc/crontab, the default user is root and the user definitely doesn’t have access to the conda.

There are two solutions I found.

  1. Load conda init script etc/profile.d/conda.sh

set -e

source "{your_conda_base}/etc/profile.d/conda.sh"
conda activate {your_conda_environment}
# Rest of your scipts goes


# or conda run
# conda run -n {your_conda_environment} {your_batch_script}.sh
  1. Use login for cron

# add to `crontab`
40 1 * * * bash -l -c "$PROJECT_ROOT/scripts/cron/run_batch_crawl.sh"

Example cron script

The following is a setup_cron.sh to install, remove, show, and test the cron job.

#!/bin/bash

# Batch Crawl Cron Job Setup Script
# This script helps install, remove, and manage the cron job

set -e

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")"
CRON_ENTRY="0 16 * * * bash -l -c $PROJECT_ROOT/scripts/cron/run_batch_crawl.sh"

# colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Log function
log() {
    echo -e "${GREEN}[$(date '+%Y-%m-%d %H:%M:%S')]${NC} $1" 
}

error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

warninig() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

info() {
    echo -e "${BLUE}[INFO]${NC} $1"
}

# Funtion to check if cron job exists
cron_job_exists() {
    crontab -l 2>/dev/null | grep -q "run_batch_crawl.sh"
}

# Function to install cron job
install_cron() {
    log "Installing batch crawl cron job..."
    
    # Check if the wrapper script exsits
    if [ ! -f "$PROJECT_ROOT/scripts/cron/run_batch_crawl.sh"]; then
        error "Wrapper script not found: $PROJECT_ROOT/scripts/cron/run_batch_crawl.sh"
        exit 1
    fi
    
    # Check if the script is executable
    if [ ! -x "$PROJECT_ROOT/scripts/cron/run_batch_crawl.sh"]; then
        warning "Making wrapper script executable"
        chmod +x "$PROJECT_ROOT/scripts/cron/run_batch_crawl.sh"
    fi
    
    # check if cron job already exists
    if cron_job_exists; then
        warning "Cron job already exists. Removing old entry..."
        remove_cron
    fi

    # Add new cron job while keeping the existing jobs by `crontab -l 2>/dev/null
    (crontab -l 2>/dev/null; "$CRON_ENTRY") | crontab -
 
    if cron_job_exists; then
        log "Cron job installed successfully!"
        log "Schedule: 1 AM KST daily"
        log "Command: $CRON_ENTRY"
    else
        error "Failed to install cron job"
        exit 1
    fi
}

# Function to remove cron job
remove_cron() {
    log "Removing batch crawl cron job..."

    if cron_job_exists; then
        crontab -l 2>/dev/null | grep -v "run_batch_crawl.sh" | crontab -
    else
        warning "No cron job found to remove"
    fi
}

# Function to show current cron jobs
show_cron() {
    log "Currnent cron jobs:"
    if crontab -l 2>/dev/null | grep -q "run_batch_crawl.sh"; then
        crontab -l | grep "run_batch_crawl.sh"
    else
        info "No batch crawl can job found"
    fi

    echo
    log "All cron jobs:"
    crontab -l 2>/dev/null || echo "No cron jobs found"
}

# Function to test the wrapper script
test_script() {
    log "Testing wrapper script..."

    if [ ! -f "$PROJECT_ROOT/scripts/cron/run_batch_crawl.sh"]; then
        error "Wrapper script not found"
        exit 1
    fi

    if [ ! -x "$PROJECT_ROOT/scripts/cron/run_batch_crawl.sh"]; then
        error "Wrapper script is not executable"
        exit 1
    fi

    log "Wrapper script exists and is executable"
    log "Script path: $PROJECT_ROOT/scripts/cron/run_batch_crawl.sh"

    # Check if conda environment exists
    if conda env list | grep -q "^customer-insight-dev "; then
        log "Conda environment 'customer-insight-dev' exists"
    else
        warning "Conda evnrionment 'customer-insight-dev' not found"
        info "Available environments:"
        conda env list
    fi
}

show_help() {
    echo "Batch Crawl Cron Job Setup Script"
    echo
    echo "Usage: $0 [COMMAND]"
    echo "Commands:"
    echo "  install    Install the cron job (runs at 1 AM KST daily)"
    echo "  remove     Revmoe the cron job"
    echo "  show       Show current cron jbos"
    echo "  test       Test the wrapper script"
    echo "  help       Show this help message"
    echo
    echo "Examples:"
    echo "  $0 install    # Install the cron job"
    echo "  $0 remove     # Remove the cron job"
    echo "  $0 show       # Show current cron jobs"
    echo "  $0 test       # Test the wrapper script"
    echo
    echo "The cron job will run the batch crawl at 1 AM KST daily"
    echo "Logs will be saved to: $PROJECT_ROOT/logs/"
}

# Main script logic
case "${1:-help}" in
    install)
        install_cron
        ;;
    remove)
        remove_cron
        ;;
    show)
        show_cron
        ;;
    test)
        test_cron
        ;;
    help|--help|-h)
        show_help
        ;;
    *)
        error "Unknown command: $1"
        echo
        show_help
        exit 1
        ;;
esac

How to use it?

$ ./scripts/cron/setup_cron.sh install
$ ./scripts/cron/setup_cron.sh show