#include "fdfs_client.h"
#include "logger.h"
#define MAX_THREADS 1024
#define MAX_LATENCIES 1000000
#define DEFAULT_FILE_SIZE 1048576 // 1MB
typedef struct {
int thread_id;
int files_to_upload;
int file_size;
char *tracker_server;
// Results
int successful_uploads;
int failed_uploads;
double *latencies;
int latency_count;
double total_bytes;
double total_time;
} thread_context_t;
typedef struct {
double mean;
double median;
double p50;
double p75;
double p90;
double p95;
double p99;
double p999;
double min;
double max;
double stddev;
} latency_stats_t;
// Global configuration
static int g_thread_count = 1;
static int g_file_count = 100;
static int g_file_size = DEFAULT_FILE_SIZE;
static char g_tracker_server[256] = "127.0.0.1:22122";
static char g_config_file[256] = "config/benchmark.conf";
static int g_warmup_enabled = 1;
static int g_warmup_duration = 10;
static int g_verbose = 0;
// Global statistics
static pthread_mutex_t g_stats_mutex = PTHREAD_MUTEX_INITIALIZER;
static int g_total_uploads = 0;
static int g_successful_uploads = 0;
static int g_failed_uploads = 0;
static double g_total_bytes = 0;
static double *g_all_latencies = NULL;
static int g_latency_count = 0;
/**
* Get current time in microseconds
*/
static double get_time_us() {
struct timeval tv;
gettimeofday(&tv, NULL);
return tv.tv_sec * 1000000.0 + tv.tv_usec;
}
/**
* Generate random data for file upload
*/
static char* generate_random_data(int size) {
char *data = (char*)malloc(size);
if (data == NULL) {
return NULL;
}
for (int i = 0; i < size; i++) {
data[i] = (char)(rand() % 256);
}
return data;
}
/**
* Compare function for qsort (for percentile calculation)
*/
static int compare_double(const void *a, const void *b) {
double diff = *(double*)a - *(double*)b;
return (diff > 0) ? 1 : (diff < 0) ? -1 : 0;
}
/**
* Calculate latency statistics
*/
static void calculate_latency_stats(double *latencies, int count, latency_stats_t *stats) {
if (count == 0) {
memset(stats, 0, sizeof(latency_stats_t));
return;
}
// Sort latencies
qsort(latencies, count, sizeof(double), compare_double);
// Calculate percentiles
stats->min = latencies[0];
stats->max = latencies[count - 1];
stats->p50 = stats->median = latencies[(int)(count * 0.50)];
stats->p75 = latencies[(int)(count * 0.75)];
stats->p90 = latencies[(int)(count * 0.90)];
stats->p95 = latencies[(int)(count * 0.95)];
stats->p99 = latencies[(int)(count * 0.99)];
stats->p999 = latencies[(int)(count * 0.999)];
// Calculate mean
double sum = 0;
for (int i = 0; i < count; i++) {
sum += latencies[i];
}
stats->mean = sum / count;
// Calculate standard deviation
double variance = 0;
for (int i = 0; i < count; i++) {
double diff = latencies[i] - stats->mean;
variance += diff * diff;
}
stats->stddev = sqrt(variance / count);
}
/**
* Upload thread function
*/
static void* upload_thread(void *arg) {
thread_context_t *ctx = (thread_context_t*)arg;
FDFSClient client;
char group_name[FDFS_GROUP_NAME_MAX_LEN + 1];
char remote_filename[256];
char *file_data = NULL;
int result;
// Initialize client
if (fdfs_client_init(ctx->tracker_server) != 0) {
fprintf(stderr, "Thread %d: Failed to initialize FDFS client\n", ctx->thread_id);
return NULL;
}
// Allocate latency array
ctx->latencies = (double*)malloc(ctx->files_to_upload * sizeof(double));
if (ctx->latencies == NULL) {
fprintf(stderr, "Thread %d: Failed to allocate latency array\n", ctx->thread_id);
fdfs_client_destroy();
return NULL;
}
ctx->latency_count = 0;
ctx->successful_uploads = 0;
ctx->failed_uploads = 0;
ctx->total_bytes = 0;
// Generate file data once
file_data = generate_random_data(ctx->file_size);
if (file_data == NULL) {
fprintf(stderr, "Thread %d: Failed to generate file data\n", ctx->thread_id);
free(ctx->latencies);
fdfs_client_destroy();
return NULL;
}
double thread_start = get_time_us();
// Upload files
for (int i = 0; i < ctx->files_to_upload; i++) {
double start_time = get_time_us();
// Upload file
result = fdfs_upload_by_buffer(file_data, ctx->file_size,
NULL, group_name, remote_filename);
double end_time = get_time_us();
double latency_ms = (end_time - start_time) / 1000.0;
if (result == 0) {
ctx->successful_uploads++;
ctx->total_bytes += ctx->file_size;
ctx->latencies[ctx->latency_count++] = latency_ms;
if (g_verbose && (i % 100 == 0)) {
printf("Thread %d: Uploaded %d/%d files (%.2f ms)\n",
ctx->thread_id, i + 1, ctx->files_to_upload, latency_ms);
}
} else {
ctx->failed_uploads++;
if (g_verbose) {
fprintf(stderr, "Thread %d: Upload failed: %s\n",
ctx->thread_id, strerror(errno));
}
}
}
double thread_end = get_time_us();
ctx->total_time = (thread_end - thread_start) / 1000000.0;
// Cleanup
free(file_data);
fdfs_client_destroy();
// Update global statistics
pthread_mutex_lock(&g_stats_mutex);
g_successful_uploads += ctx->successful_uploads;
g_failed_uploads += ctx->failed_uploads;
g_total_bytes += ctx->total_bytes;
pthread_mutex_unlock(&g_stats_mutex);
return NULL;
}
/**
* Run warmup phase
*/
static void run_warmup() {
if (!g_warmup_enabled || g_warmup_duration <= 0) {
return;
}
printf("Running warmup phase for %d seconds...\n", g_warmup_duration);
thread_context_t warmup_ctx;
warmup_ctx.thread_id = 0;
warmup_ctx.files_to_upload = 10;
warmup_ctx.file_size = g_file_size;
warmup_ctx.tracker_server = g_tracker_server;
pthread_t warmup_thread;
pthread_create(&warmup_thread, NULL, upload_thread, &warmup_ctx);
sleep(g_warmup_duration);
pthread_join(warmup_thread, NULL);
if (warmup_ctx.latencies) {
free(warmup_ctx.latencies);
}
printf("Warmup complete.\n\n");
}
/**
* Print results in JSON format
*/
static void print_results(double total_time, latency_stats_t *stats) {
double throughput_mbps = (g_total_bytes / (1024 * 1024)) / total_time;
double iops = g_successful_uploads / total_time;
double success_rate = (double)g_successful_uploads / g_total_uploads * 100.0;
printf("{\n");
printf(" \"benchmark\": \"upload\",\n");
printf(" \"timestamp\": \"%ld\",\n", time(NULL));
printf(" \"configuration\": {\n");
printf(" \"threads\": %d,\n", g_thread_count);
printf(" \"file_count\": %d,\n", g_file_count);
printf(" \"file_size\": %d,\n", g_file_size);
printf(" \"tracker_server\": \"%s\"\n", g_tracker_server);
printf(" },\n");
printf(" \"metrics\": {\n");
printf(" \"throughput_mbps\": %.2f,\n", throughput_mbps);
printf(" \"iops\": %.2f,\n", iops);
printf(" \"latency_ms\": {\n");
printf(" \"mean\": %.2f,\n", stats->mean);
printf(" \"median\": %.2f,\n", stats->median);
printf(" \"p50\": %.2f,\n", stats->p50);
printf(" \"p75\": %.2f,\n", stats->p75);
printf(" \"p90\": %.2f,\n", stats->p90);
printf(" \"p95\": %.2f,\n", stats->p95);
printf(" \"p99\": %.2f,\n", stats->p99);
printf(" \"p999\": %.2f,\n", stats->p999);
printf(" \"min\": %.2f,\n", stats->min);
printf(" \"max\": %.2f,\n", stats->max);
printf(" \"stddev\": %.2f\n", stats->stddev);
printf(" },\n");
printf(" \"operations\": {\n");
printf(" \"total\": %d,\n", g_total_uploads);
printf(" \"successful\": %d,\n", g_successful_uploads);
printf(" \"failed\": %d,\n", g_failed_uploads);
printf(" \"success_rate\": %.2f\n", success_rate);
printf(" },\n");
printf(" \"duration_seconds\": %.2f,\n", total_time);
printf(" \"total_mb\": %.2f\n", g_total_bytes / (1024 * 1024));
printf(" }\n");
printf("}\n");
}
/**
* Print usage information
*/
static void print_usage(const char *program) {
printf("Usage: %s [OPTIONS]\n", program);
printf("\nOptions:\n");
printf(" -t, --threads NUM Number of concurrent threads (default: 1)\n");
printf(" -f, --files NUM Number of files to upload (default: 100)\n");
printf(" -s, --size BYTES File size in bytes (default: 1048576)\n");
printf(" -c, --config FILE Configuration file (default: config/benchmark.conf)\n");
printf(" -T, --tracker SERVER Tracker server (default: 127.0.0.1:22122)\n");
printf(" -w, --warmup SECONDS Warmup duration (default: 10, 0 to disable)\n");
printf(" -v, --verbose Enable verbose output\n");
printf(" -h, --help Show this help message\n");
}
/**
* Main function
*/
int main(int argc, char *argv[]) {
pthread_t threads[MAX_THREADS];
thread_context_t contexts[MAX_THREADS];
// Parse command line arguments
static struct option long_options[] = {
{"threads", required_argument, 0, 't'},
{"files", required_argument, 0, 'f'},
{"size", required_argument, 0, 's'},
{"config", required_argument, 0, 'c'},
{"tracker", required_argument, 0, 'T'},
{"warmup", required_argument, 0, 'w'},
{"verbose", no_argument, 0, 'v'},
{"help", no_argument, 0, 'h'},
{0, 0, 0, 0}
};
int opt;
while ((opt = getopt_long(argc, argv, "t:f:s:c:T:w:vh", long_options, NULL)) != -1) {
switch (opt) {
case 't':
g_thread_count = atoi(optarg);
break;
case 'f':
g_file_count = atoi(optarg);
break;
case 's':
g_file_size = atoi(optarg);
break;
case 'c':
strncpy(g_config_file, optarg, sizeof(g_config_file) - 1);
break;
case 'T':
strncpy(g_tracker_server, optarg, sizeof(g_tracker_server) - 1);
break;
case 'w':
g_warmup_duration = atoi(optarg);
break;
case 'v':
g_verbose = 1;
break;
case 'h':
print_usage(argv[0]);
return 0;
default:
print_usage(argv[0]);
return 1;
}
}
// Validate parameters
if (g_thread_count < 1 || g_thread_count > MAX_THREADS) {
fprintf(stderr, "Error: Thread count must be between 1 and %d\n", MAX_THREADS);
return 1;
}
if (g_file_count < 1) {
fprintf(stderr, "Error: File count must be at least 1\n");
return 1;
}
if (g_file_size < 1) {
fprintf(stderr, "Error: File size must be at least 1 byte\n");
return 1;
}
printf("FastDFS Upload Performance Benchmark\n");
printf("=====================================\n");
printf("Threads: %d\n", g_thread_count);
printf("Files per thread: %d\n", g_file_count / g_thread_count);
printf("File size: %d bytes (%.2f MB)\n", g_file_size, g_file_size / (1024.0 * 1024.0));
printf("Total files: %d\n", g_file_count);
printf("Tracker: %s\n\n", g_tracker_server);
// Initialize random seed
srand(time(NULL));
// Run warmup
run_warmup();
// Allocate global latency array
g_all_latencies = (double*)malloc(g_file_count * sizeof(double));
if (g_all_latencies == NULL) {
fprintf(stderr, "Error: Failed to allocate latency array\n");
return 1;
}
// Initialize thread contexts
int files_per_thread = g_file_count / g_thread_count;
int remaining_files = g_file_count % g_thread_count;
for (int i = 0; i < g_thread_count; i++) {
contexts[i].thread_id = i;
contexts[i].files_to_upload = files_per_thread + (i < remaining_files ? 1 : 0);
contexts[i].file_size = g_file_size;
contexts[i].tracker_server = g_tracker_server;
}
// Start benchmark
printf("Starting benchmark...\n");
double start_time = get_time_us();
// Create threads
for (int i = 0; i < g_thread_count; i++) {
if (pthread_create(&threads[i], NULL, upload_thread, &contexts[i]) != 0) {
fprintf(stderr, "Error: Failed to create thread %d\n", i);
return 1;
}
}
// Wait for threads to complete
for (int i = 0; i < g_thread_count; i++) {
pthread_join(threads[i], NULL);
}
double end_time = get_time_us();
double total_time = (end_time - start_time) / 1000000.0;
// Collect all latencies
g_latency_count = 0;
for (int i = 0; i < g_thread_count; i++) {
for (int j = 0; j < contexts[i].latency_count; j++) {
g_all_latencies[g_latency_count++] = contexts[i].latencies[j];
}
free(contexts[i].latencies);
}
// Calculate statistics
latency_stats_t stats;
calculate_latency_stats(g_all_latencies, g_latency_count, &stats);
// Print results
printf("\nBenchmark complete!\n\n");
print_results(total_time, &stats);
// Cleanup
free(g_all_latencies);
return 0;
}
================================================
FILE: benchmarks/config/benchmark.conf
================================================
# FastDFS Benchmark Configuration
# Tracker Server Configuration
tracker_server = 127.0.0.1:22122
# Storage Server Configuration (optional, for direct storage testing)
storage_server = 127.0.0.1:23000
# Connection Settings
connect_timeout = 30
network_timeout = 60
max_connections = 256
# Benchmark Settings
[upload]
# Number of concurrent threads
threads = 10
# Number of files to upload
file_count = 1000
# Default file size in bytes (1MB)
file_size = 1048576
# Enable warm-up phase
warmup_enabled = true
# Warm-up duration in seconds
warmup_duration = 10
[download]
# Number of concurrent threads
threads = 10
# Number of download iterations
iterations = 1000
# Enable warm-up phase
warmup_enabled = true
# Warm-up duration in seconds
warmup_duration = 10
[concurrent]
# Number of concurrent users to simulate
concurrent_users = 50
# Test duration in seconds
duration = 300
# Operation mix (upload:download:delete ratio)
operation_mix = 50:45:5
# Think time between operations (ms)
think_time = 100
[small_files]
# Number of small files to test
file_count = 10000
# Minimum file size (1KB)
min_size = 1024
# Maximum file size (100KB)
max_size = 102400
# Number of concurrent threads
threads = 20
[large_files]
# Number of large files to test
file_count = 100
# Minimum file size (100MB)
min_size = 104857600
# Maximum file size (1GB)
max_size = 1073741824
# Number of concurrent threads
threads = 5
[metadata]
# Number of metadata operations
operation_count = 10000
# Number of concurrent threads
threads = 10
# Operation types (query:update:delete ratio)
operation_types = 70:20:10
# Resource Monitoring
[monitoring]
# Enable resource monitoring
enabled = true
# Monitoring interval in seconds
interval = 1
# Metrics to collect (cpu,memory,network,disk)
metrics = cpu,memory,network,disk
# Result Settings
[results]
# Output directory for results
output_dir = results/
# Result format (json, csv, both)
format = json
# Enable detailed logging
detailed_logging = true
# Log file path
log_file = results/benchmark.log
# Report Settings
[report]
# Report format (html, pdf, both)
format = html
# Include graphs
include_graphs = true
# Include resource utilization charts
include_resources = true
# Report template
template = default
# Advanced Settings
[advanced]
# Enable latency histogram
latency_histogram = true
# Histogram buckets (ms)
histogram_buckets = 1,5,10,20,50,100,200,500,1000
# Enable per-thread statistics
per_thread_stats = false
# Enable real-time progress display
show_progress = true
# Progress update interval (seconds)
progress_interval = 5
================================================
FILE: benchmarks/results/template.json
================================================
{
"benchmark": "template",
"version": "1.0.0",
"fastdfs_version": "6.12.1",
"timestamp": "2024-01-15T10:30:00Z",
"hostname": "benchmark-server-01",
"system_info": {
"os": "Linux",
"kernel": "5.15.0",
"cpu_model": "Intel Xeon E5-2680 v4",
"cpu_cores": 28,
"memory_gb": 64,
"disk_type": "SSD",
"network_speed_gbps": 10
},
"configuration": {
"tracker_server": "127.0.0.1:22122",
"storage_server": "127.0.0.1:23000",
"threads": 10,
"file_count": 1000,
"file_size": 1048576,
"warmup_enabled": true,
"warmup_duration": 10
},
"metrics": {
"throughput": {
"upload_mbps": 0.0,
"download_mbps": 0.0,
"total_mbps": 0.0
},
"iops": {
"read": 0,
"write": 0,
"total": 0
},
"latency_ms": {
"mean": 0.0,
"median": 0.0,
"p50": 0.0,
"p75": 0.0,
"p90": 0.0,
"p95": 0.0,
"p99": 0.0,
"p999": 0.0,
"min": 0.0,
"max": 0.0,
"stddev": 0.0
},
"operations": {
"total": 0,
"successful": 0,
"failed": 0,
"timeout": 0,
"success_rate": 0.0,
"error_rate": 0.0,
"timeout_rate": 0.0
},
"duration": {
"total_seconds": 0.0,
"warmup_seconds": 0.0,
"test_seconds": 0.0
},
"data_transferred": {
"total_bytes": 0,
"total_mb": 0.0,
"total_gb": 0.0
}
},
"resource_utilization": {
"cpu": {
"mean_percent": 0.0,
"max_percent": 0.0,
"min_percent": 0.0
},
"memory": {
"mean_mb": 0.0,
"max_mb": 0.0,
"min_mb": 0.0,
"mean_percent": 0.0
},
"network": {
"rx_mbps": 0.0,
"tx_mbps": 0.0,
"total_mbps": 0.0,
"rx_packets": 0,
"tx_packets": 0
},
"disk": {
"read_mbps": 0.0,
"write_mbps": 0.0,
"read_iops": 0,
"write_iops": 0,
"utilization_percent": 0.0
}
},
"latency_histogram": {
"buckets": [1, 5, 10, 20, 50, 100, 200, 500, 1000],
"counts": [0, 0, 0, 0, 0, 0, 0, 0, 0]
},
"per_thread_stats": [],
"errors": [],
"warnings": [],
"notes": ""
}
================================================
FILE: benchmarks/scripts/compare_versions.py
================================================
#!/usr/bin/env python3
"""
FastDFS Version Comparison Tool
Compares performance between two FastDFS versions
"""
import json
import argparse
import sys
from pathlib import Path
def load_results(filepath):
"""Load benchmark results from JSON file"""
try:
with open(filepath, 'r') as f:
return json.load(f)
except FileNotFoundError:
print(f"Error: File not found: {filepath}", file=sys.stderr)
sys.exit(1)
except json.JSONDecodeError as e:
print(f"Error: Invalid JSON in {filepath}: {e}", file=sys.stderr)
sys.exit(1)
def calculate_change(baseline, current):
"""Calculate percentage change"""
if baseline == 0:
return 0
return ((current - baseline) / baseline) * 100
def format_change(change):
"""Format change with color indicator"""
if change > 0:
return f"+{change:.2f}% ⬆️"
elif change < 0:
return f"{change:.2f}% ⬇️"
else:
return "0.00% ➡️"
def compare_metrics(baseline_metrics, current_metrics, metric_name):
"""Compare a specific metric between versions"""
baseline_val = baseline_metrics.get(metric_name, 0)
current_val = current_metrics.get(metric_name, 0)
change = calculate_change(baseline_val, current_val)
return {
'baseline': baseline_val,
'current': current_val,
'change': change,
'change_str': format_change(change)
}
def compare_benchmark(baseline_data, current_data, bench_name):
"""Compare a specific benchmark between versions"""
print(f"\n{'='*80}")
print(f" {bench_name.upper()} BENCHMARK")
print(f"{'='*80}")
if 'metrics' not in baseline_data or 'metrics' not in current_data:
print(" ⚠️ Insufficient data for comparison")
return
baseline_metrics = baseline_data['metrics']
current_metrics = current_data['metrics']
# Compare throughput
if 'throughput_mbps' in baseline_metrics and 'throughput_mbps' in current_metrics:
comp = compare_metrics(baseline_metrics, current_metrics, 'throughput_mbps')
print(f"\n Throughput:")
print(f" Baseline: {comp['baseline']:.2f} MB/s")
print(f" Current: {comp['current']:.2f} MB/s")
print(f" Change: {comp['change_str']}")
# Compare IOPS
if 'iops' in baseline_metrics and 'iops' in current_metrics:
comp = compare_metrics(baseline_metrics, current_metrics, 'iops')
print(f"\n IOPS:")
print(f" Baseline: {comp['baseline']:.2f} ops/s")
print(f" Current: {comp['current']:.2f} ops/s")
print(f" Change: {comp['change_str']}")
# Compare latency
if 'latency_ms' in baseline_metrics and 'latency_ms' in current_metrics:
baseline_latency = baseline_metrics['latency_ms']
current_latency = current_metrics['latency_ms']
print(f"\n Latency (ms):")
for metric in ['mean', 'p50', 'p95', 'p99']:
if metric in baseline_latency and metric in current_latency:
baseline_val = baseline_latency[metric]
current_val = current_latency[metric]
change = calculate_change(baseline_val, current_val)
# For latency, lower is better, so invert the indicator
if change < 0:
indicator = "⬆️ (better)"
elif change > 0:
indicator = "⬇️ (worse)"
else:
indicator = "➡️"
print(f" {metric.upper():6s}: {baseline_val:8.2f} → {current_val:8.2f} ({change:+.2f}%) {indicator}")
# Compare success rate
if 'operations' in baseline_metrics and 'operations' in current_metrics:
baseline_ops = baseline_metrics['operations']
current_ops = current_metrics['operations']
if 'success_rate' in baseline_ops and 'success_rate' in current_ops:
baseline_rate = baseline_ops['success_rate']
current_rate = current_ops['success_rate']
change = current_rate - baseline_rate
print(f"\n Success Rate:")
print(f" Baseline: {baseline_rate:.2f}%")
print(f" Current: {current_rate:.2f}%")
print(f" Change: {change:+.2f}% {'⬆️' if change > 0 else '⬇️' if change < 0 else '➡️'}")
def generate_html_comparison(baseline, current, output_file):
"""Generate HTML comparison report"""
html = """
FastDFS Version Comparison
🔄 FastDFS Version Comparison
Comparison Details
Baseline Version: {baseline_version}
Current Version: {current_version}
Baseline Date: {baseline_date}
Current Date: {current_date}
{comparison_tables}
Summary
This comparison shows the performance differences between two versions of FastDFS.
Green values indicate improvements, red values indicate regressions.
"""
baseline_version = baseline.get('fastdfs_version', 'Unknown')
current_version = current.get('fastdfs_version', 'Unknown')
baseline_date = baseline.get('timestamp', 'Unknown')
current_date = current.get('timestamp', 'Unknown')
comparison_tables = ""
baseline_results = baseline.get('results', {})
current_results = current.get('results', {})
for bench_name in baseline_results.keys():
if bench_name not in current_results:
continue
baseline_data = baseline_results[bench_name]
current_data = current_results[bench_name]
if 'metrics' not in baseline_data or 'metrics' not in current_data:
continue
baseline_metrics = baseline_data['metrics']
current_metrics = current_data['metrics']
comparison_tables += f"{bench_name.replace('_', ' ').title()}
"
comparison_tables += "| Metric | Baseline | Current | Change |
|---|
"
# Compare key metrics
metrics_to_compare = [
('throughput_mbps', 'Throughput (MB/s)', True),
('iops', 'IOPS', True),
]
for metric_key, metric_label, higher_is_better in metrics_to_compare:
if metric_key in baseline_metrics and metric_key in current_metrics:
baseline_val = baseline_metrics[metric_key]
current_val = current_metrics[metric_key]
change = calculate_change(baseline_val, current_val)
if higher_is_better:
css_class = 'improvement' if change > 0 else 'regression' if change < 0 else 'neutral'
else:
css_class = 'regression' if change > 0 else 'improvement' if change < 0 else 'neutral'
comparison_tables += f"""
| {metric_label} |
{baseline_val:.2f} |
{current_val:.2f} |
{change:+.2f}% |
"""
# Add latency comparison
if 'latency_ms' in baseline_metrics and 'latency_ms' in current_metrics:
baseline_latency = baseline_metrics['latency_ms']
current_latency = current_metrics['latency_ms']
for lat_metric in ['mean', 'p95', 'p99']:
if lat_metric in baseline_latency and lat_metric in current_latency:
baseline_val = baseline_latency[lat_metric]
current_val = current_latency[lat_metric]
change = calculate_change(baseline_val, current_val)
# For latency, lower is better
css_class = 'improvement' if change < 0 else 'regression' if change > 0 else 'neutral'
comparison_tables += f"""
| Latency {lat_metric.upper()} (ms) |
{baseline_val:.2f} |
{current_val:.2f} |
{change:+.2f}% |
"""
comparison_tables += "
"
final_html = html.format(
baseline_version=baseline_version,
current_version=current_version,
baseline_date=baseline_date,
current_date=current_date,
comparison_tables=comparison_tables
)
with open(output_file, 'w') as f:
f.write(final_html)
print(f"\n✓ HTML comparison report generated: {output_file}")
def main():
parser = argparse.ArgumentParser(
description='Compare FastDFS benchmark results between versions'
)
parser.add_argument(
'-b', '--baseline',
required=True,
help='Baseline benchmark results (JSON)'
)
parser.add_argument(
'-c', '--current',
required=True,
help='Current benchmark results (JSON)'
)
parser.add_argument(
'-o', '--output',
default='comparison.html',
help='Output HTML file (default: comparison.html)'
)
parser.add_argument(
'--text-only',
action='store_true',
help='Only print text comparison (no HTML)'
)
args = parser.parse_args()
# Load results
print(f"Loading baseline results from {args.baseline}...")
baseline = load_results(args.baseline)
print(f"Loading current results from {args.current}...")
current = load_results(args.current)
print("\n" + "="*80)
print(" FASTDFS VERSION COMPARISON")
print("="*80)
baseline_version = baseline.get('fastdfs_version', 'Unknown')
current_version = current.get('fastdfs_version', 'Unknown')
print(f"\n Baseline: {baseline_version} ({baseline.get('timestamp', 'Unknown')})")
print(f" Current: {current_version} ({current.get('timestamp', 'Unknown')})")
# Compare each benchmark
baseline_results = baseline.get('results', {})
current_results = current.get('results', {})
for bench_name in baseline_results.keys():
if bench_name in current_results:
compare_benchmark(
baseline_results[bench_name],
current_results[bench_name],
bench_name
)
# Generate HTML report
if not args.text_only:
print(f"\n{'='*80}")
print(" Generating HTML report...")
generate_html_comparison(baseline, current, args.output)
print(f"\n{'='*80}")
print(" Comparison complete!")
print(f"{'='*80}\n")
if __name__ == '__main__':
main()
================================================
FILE: benchmarks/scripts/generate_report.py
================================================
#!/usr/bin/env python3
"""
FastDFS Benchmark Report Generator
Generates HTML/PDF reports from benchmark results
"""
import json
import argparse
import sys
from datetime import datetime
from pathlib import Path
import statistics
# HTML template
HTML_TEMPLATE = """
FastDFS Benchmark Report
📊 FastDFS Performance Benchmark Report
Executive Summary
{summary_metrics}
{benchmark_sections}
"""
def load_results(input_file):
"""Load benchmark results from JSON file"""
try:
with open(input_file, 'r') as f:
return json.load(f)
except FileNotFoundError:
print(f"Error: File not found: {input_file}", file=sys.stderr)
sys.exit(1)
except json.JSONDecodeError as e:
print(f"Error: Invalid JSON in {input_file}: {e}", file=sys.stderr)
sys.exit(1)
def format_bytes(bytes_val):
"""Format bytes to human readable format"""
for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
if bytes_val < 1024.0:
return f"{bytes_val:.2f} {unit}"
bytes_val /= 1024.0
return f"{bytes_val:.2f} PB"
def format_number(num):
"""Format number with thousands separator"""
return f"{num:,.2f}"
def create_metric_card(label, value, unit="", card_type="info"):
"""Create a metric card HTML"""
return f"""
"""
def create_progress_bar(label, percentage):
"""Create a progress bar HTML"""
return f"""
"""
def create_latency_table(latency_data):
"""Create latency statistics table"""
if not latency_data:
return "No latency data available
"
html = """
| Metric |
Value (ms) |
"""
metrics = [
('Mean', 'mean'),
('Median (p50)', 'median'),
('p75', 'p75'),
('p90', 'p90'),
('p95', 'p95'),
('p99', 'p99'),
('Min', 'min'),
('Max', 'max'),
('Std Dev', 'stddev')
]
for label, key in metrics:
value = latency_data.get(key, 0)
html += f"| {label} | {value:.2f} |
\n"
html += "
"
return html
def generate_upload_section(data):
"""Generate upload benchmark section"""
if 'metrics' not in data:
return ""
metrics = data['metrics']
config = data.get('configuration', {})
html = """
📤 Upload Performance
Configuration: {threads} threads, {file_count} files, {file_size} file size
""".format(
threads=config.get('threads', 'N/A'),
file_count=config.get('file_count', 'N/A'),
file_size=format_bytes(config.get('file_size', 0))
)
html += create_metric_card(
"Throughput",
format_number(metrics.get('throughput_mbps', 0)),
"MB/s",
"success"
)
html += create_metric_card(
"IOPS",
format_number(metrics.get('iops', 0)),
"ops/s",
"info"
)
ops = metrics.get('operations', {})
success_rate = ops.get('success_rate', 0)
html += create_metric_card(
"Success Rate",
f"{success_rate:.1f}",
"%",
"success" if success_rate > 95 else "warning"
)
html += create_metric_card(
"Duration",
format_number(metrics.get('duration_seconds', 0)),
"seconds",
"info"
)
html += "
"
# Latency statistics
if 'latency_ms' in metrics:
html += "
Latency Statistics
"
html += create_latency_table(metrics['latency_ms'])
# Operations breakdown
if 'operations' in metrics:
ops = metrics['operations']
html += "
Operations
"
html += create_progress_bar(
f"Successful: {ops.get('successful', 0)} / {ops.get('total', 0)}",
ops.get('success_rate', 0)
)
html += "
"
return html
def generate_download_section(data):
"""Generate download benchmark section"""
if 'metrics' not in data:
return ""
metrics = data['metrics']
config = data.get('configuration', {})
html = """
📥 Download Performance
Configuration: {threads} threads, {iterations} iterations
""".format(
threads=config.get('threads', 'N/A'),
iterations=config.get('iterations', 'N/A')
)
html += create_metric_card(
"Throughput",
format_number(metrics.get('throughput_mbps', 0)),
"MB/s",
"success"
)
html += create_metric_card(
"IOPS",
format_number(metrics.get('iops', 0)),
"ops/s",
"info"
)
ops = metrics.get('operations', {})
success_rate = ops.get('success_rate', 0)
html += create_metric_card(
"Success Rate",
f"{success_rate:.1f}",
"%",
"success" if success_rate > 95 else "warning"
)
html += "
"
if 'latency_ms' in metrics:
html += "
Latency Statistics
"
html += create_latency_table(metrics['latency_ms'])
html += "
"
return html
def generate_concurrent_section(data):
"""Generate concurrent operations section"""
if 'metrics' not in data:
return ""
metrics = data['metrics']
config = data.get('configuration', {})
html = """
👥 Concurrent Operations
Configuration: {users} users, {duration}s duration, mix: {mix}
""".format(
users=config.get('users', 'N/A'),
duration=config.get('duration', 'N/A'),
mix=config.get('operation_mix', 'N/A')
)
ops = metrics.get('operations', {})
html += create_metric_card(
"Total Operations",
format_number(ops.get('total', 0)),
"ops",
"info"
)
html += create_metric_card(
"Ops/Second",
format_number(ops.get('per_second', 0)),
"ops/s",
"success"
)
html += "
"
# Operation breakdown
html += "
Operation Breakdown
"
if 'uploads' in ops:
uploads = ops['uploads']
html += create_progress_bar(
f"Uploads: {uploads.get('successful', 0)} / {uploads.get('total', 0)}",
uploads.get('success_rate', 0)
)
if 'downloads' in ops:
downloads = ops['downloads']
html += create_progress_bar(
f"Downloads: {downloads.get('successful', 0)} / {downloads.get('total', 0)}",
downloads.get('success_rate', 0)
)
if 'deletes' in ops:
deletes = ops['deletes']
html += create_progress_bar(
f"Deletes: {deletes.get('successful', 0)} / {deletes.get('total', 0)}",
deletes.get('success_rate', 0)
)
html += "
"
return html
def generate_small_files_section(data):
"""Generate small files section"""
if 'metrics' not in data:
return ""
metrics = data['metrics']
config = data.get('configuration', {})
html = """
📄 Small Files Performance
Configuration: {threads} threads, {count} files, {min_size} - {max_size}
""".format(
threads=config.get('threads', 'N/A'),
count=config.get('file_count', 'N/A'),
min_size=format_bytes(config.get('min_size', 0)),
max_size=format_bytes(config.get('max_size', 0))
)
html += create_metric_card(
"Throughput",
format_number(metrics.get('throughput_mbps', 0)),
"MB/s",
"success"
)
html += create_metric_card(
"IOPS",
format_number(metrics.get('iops', 0)),
"ops/s",
"info"
)
html += create_metric_card(
"Avg File Size",
format_number(metrics.get('avg_file_size_kb', 0)),
"KB",
"info"
)
html += "
"
if 'latency_ms' in metrics:
html += "
Latency Statistics
"
html += create_latency_table(metrics['latency_ms'])
html += "
"
return html
def generate_large_files_section(data):
"""Generate large files section"""
if 'metrics' not in data:
return ""
metrics = data['metrics']
config = data.get('configuration', {})
html = """
📦 Large Files Performance
Configuration: {threads} threads, {count} files, {min_size} - {max_size}
""".format(
threads=config.get('threads', 'N/A'),
count=config.get('file_count', 'N/A'),
min_size=format_number(config.get('min_size_mb', 0)) + " MB",
max_size=format_number(config.get('max_size_mb', 0)) + " MB"
)
html += create_metric_card(
"Throughput",
format_number(metrics.get('throughput_mbps', 0)),
"MB/s",
"success"
)
html += create_metric_card(
"Total Data",
format_number(metrics.get('total_gb', 0)),
"GB",
"info"
)
html += create_metric_card(
"Avg File Size",
format_number(metrics.get('avg_file_size_mb', 0)),
"MB",
"info"
)
html += "
"
if 'latency_seconds' in metrics:
html += "
Latency Statistics (seconds)
"
html += create_latency_table(metrics['latency_seconds'])
html += "
"
return html
def generate_metadata_section(data):
"""Generate metadata operations section"""
if 'metrics' not in data:
return ""
metrics = data['metrics']
config = data.get('configuration', {})
html = """
🏷️ Metadata Operations
Configuration: {threads} threads, {ops} operations, mix: {mix}
""".format(
threads=config.get('threads', 'N/A'),
ops=config.get('operation_count', 'N/A'),
mix=config.get('operation_mix', 'N/A')
)
html += create_metric_card(
"Total Operations",
format_number(metrics.get('total_operations', 0)),
"ops",
"info"
)
html += create_metric_card(
"Ops/Second",
format_number(metrics.get('ops_per_second', 0)),
"ops/s",
"success"
)
html += "
"
# Operation breakdown
if 'operations' in metrics:
ops = metrics['operations']
html += "
Operation Breakdown
"
for op_type in ['query', 'update', 'delete']:
if op_type in ops:
op_data = ops[op_type]
html += create_progress_bar(
f"{op_type.capitalize()}: {op_data.get('successful', 0)} / {op_data.get('total', 0)}",
op_data.get('success_rate', 0)
)
html += "
"
return html
def generate_report(results, output_file):
"""Generate HTML report"""
# Extract system info
system_info = results.get('system_info', {})
system_str = f"{system_info.get('hostname', 'Unknown')} - {system_info.get('os', 'Unknown')} {system_info.get('kernel', '')}"
# Extract configuration
config = results.get('configuration', {})
tracker = config.get('tracker_server', 'Unknown')
# Generate timestamp
generated_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
benchmark_time = results.get('timestamp', 'Unknown')
# Generate summary metrics
summary_metrics = ""
benchmark_results = results.get('results', {})
# Calculate overall metrics
total_throughput = 0
total_iops = 0
count = 0
for bench_name, bench_data in benchmark_results.items():
if 'metrics' in bench_data:
metrics = bench_data['metrics']
if 'throughput_mbps' in metrics:
total_throughput += metrics['throughput_mbps']
count += 1
if 'iops' in metrics:
total_iops += metrics['iops']
if count > 0:
summary_metrics += create_metric_card(
"Avg Throughput",
format_number(total_throughput / count),
"MB/s",
"success"
)
summary_metrics += create_metric_card(
"Total IOPS",
format_number(total_iops),
"ops/s",
"info"
)
summary_metrics += create_metric_card(
"Benchmarks Run",
str(len(benchmark_results)),
"tests",
"info"
)
# Generate benchmark sections
benchmark_sections = ""
section_generators = {
'upload': generate_upload_section,
'download': generate_download_section,
'concurrent': generate_concurrent_section,
'small_files': generate_small_files_section,
'large_files': generate_large_files_section,
'metadata': generate_metadata_section
}
for bench_name, bench_data in benchmark_results.items():
if bench_name in section_generators:
benchmark_sections += section_generators[bench_name](bench_data)
# Generate final HTML
html = HTML_TEMPLATE.format(
generated_time=generated_time,
benchmark_time=benchmark_time,
system_info=system_str,
tracker_server=tracker,
summary_metrics=summary_metrics,
benchmark_sections=benchmark_sections
)
# Write to file
with open(output_file, 'w') as f:
f.write(html)
print(f"✓ Report generated: {output_file}")
def main():
parser = argparse.ArgumentParser(
description='Generate HTML report from FastDFS benchmark results'
)
parser.add_argument(
'-i', '--input',
required=True,
help='Input JSON file with benchmark results'
)
parser.add_argument(
'-o', '--output',
default='report.html',
help='Output HTML file (default: report.html)'
)
args = parser.parse_args()
# Load results
print(f"Loading results from {args.input}...")
results = load_results(args.input)
# Generate report
print(f"Generating report...")
generate_report(results, args.output)
print(f"\n✓ Report successfully generated!")
print(f" Open {args.output} in your browser to view the report.")
if __name__ == '__main__':
main()
================================================
FILE: benchmarks/scripts/run_all_benchmarks.sh
================================================
#!/bin/bash
###############################################################################
# FastDFS Benchmark Runner
#
# Runs all benchmarks and collects results
###############################################################################
set -e
# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BENCHMARK_DIR="$(dirname "$SCRIPT_DIR")"
RESULTS_DIR="$BENCHMARK_DIR/results"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
RESULT_FILE="$RESULTS_DIR/benchmark_${TIMESTAMP}.json"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Default configuration
TRACKER_SERVER="127.0.0.1:22122"
THREADS=10
VERBOSE=0
###############################################################################
# Functions
###############################################################################
print_header() {
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}$1${NC}"
echo -e "${BLUE}========================================${NC}"
}
print_success() {
echo -e "${GREEN}✓ $1${NC}"
}
print_error() {
echo -e "${RED}✗ $1${NC}"
}
print_warning() {
echo -e "${YELLOW}⚠ $1${NC}"
}
print_info() {
echo -e "${BLUE}ℹ $1${NC}"
}
usage() {
cat << EOF
Usage: $0 [OPTIONS]
Run all FastDFS benchmarks and generate comprehensive report.
Options:
-t, --tracker SERVER Tracker server address (default: 127.0.0.1:22122)
-T, --threads NUM Number of threads (default: 10)
-o, --output FILE Output file (default: results/benchmark_TIMESTAMP.json)
-v, --verbose Enable verbose output
-h, --help Show this help message
Examples:
$0
$0 --tracker 192.168.1.100:22122 --threads 20
$0 -v -o my_results.json
EOF
}
check_prerequisites() {
print_header "Checking Prerequisites"
# Check if benchmark binaries exist
local missing=0
for bench in benchmark_upload benchmark_download benchmark_concurrent \
benchmark_small_files benchmark_large_files benchmark_metadata; do
if [ ! -f "$BENCHMARK_DIR/$bench" ]; then
print_error "Missing benchmark: $bench"
missing=1
fi
done
if [ $missing -eq 1 ]; then
print_error "Some benchmarks are missing. Please run 'make' first."
exit 1
fi
# Check if tracker is reachable
print_info "Checking tracker connectivity: $TRACKER_SERVER"
local tracker_host=$(echo $TRACKER_SERVER | cut -d: -f1)
local tracker_port=$(echo $TRACKER_SERVER | cut -d: -f2)
if command -v nc &> /dev/null; then
if nc -z -w5 $tracker_host $tracker_port 2>/dev/null; then
print_success "Tracker is reachable"
else
print_warning "Cannot reach tracker at $TRACKER_SERVER"
print_warning "Continuing anyway, but benchmarks may fail..."
fi
fi
# Create results directory
mkdir -p "$RESULTS_DIR"
print_success "Results directory: $RESULTS_DIR"
echo ""
}
run_benchmark() {
local name=$1
local binary=$2
shift 2
local args="$@"
print_header "Running $name Benchmark"
print_info "Command: $binary $args"
local output_file="$RESULTS_DIR/${name}_${TIMESTAMP}.json"
if [ $VERBOSE -eq 1 ]; then
"$BENCHMARK_DIR/$binary" $args 2>&1 | tee "$output_file"
else
"$BENCHMARK_DIR/$binary" $args > "$output_file" 2>&1
fi
local exit_code=$?
if [ $exit_code -eq 0 ]; then
print_success "$name benchmark completed"
echo "$output_file"
return 0
else
print_error "$name benchmark failed (exit code: $exit_code)"
return 1
fi
echo ""
}
combine_results() {
print_header "Combining Results"
local upload_result="$RESULTS_DIR/upload_${TIMESTAMP}.json"
local download_result="$RESULTS_DIR/download_${TIMESTAMP}.json"
local concurrent_result="$RESULTS_DIR/concurrent_${TIMESTAMP}.json"
local small_files_result="$RESULTS_DIR/small_files_${TIMESTAMP}.json"
local large_files_result="$RESULTS_DIR/large_files_${TIMESTAMP}.json"
local metadata_result="$RESULTS_DIR/metadata_${TIMESTAMP}.json"
cat > "$RESULT_FILE" << EOF
{
"benchmark_suite": "FastDFS Performance Benchmark",
"version": "1.0.0",
"timestamp": "$(date -Iseconds)",
"system_info": {
"hostname": "$(hostname)",
"os": "$(uname -s)",
"kernel": "$(uname -r)",
"cpu": "$(grep -m1 'model name' /proc/cpuinfo 2>/dev/null | cut -d: -f2 | xargs || echo 'Unknown')",
"memory_gb": $(free -g 2>/dev/null | awk '/^Mem:/{print $2}' || echo 0)
},
"configuration": {
"tracker_server": "$TRACKER_SERVER",
"threads": $THREADS
},
"results": {
EOF
# Add individual benchmark results
local first=1
for result_file in "$upload_result" "$download_result" "$concurrent_result" \
"$small_files_result" "$large_files_result" "$metadata_result"; do
if [ -f "$result_file" ]; then
if [ $first -eq 0 ]; then
echo "," >> "$RESULT_FILE"
fi
first=0
local bench_name=$(basename "$result_file" "_${TIMESTAMP}.json")
echo " \"$bench_name\": " >> "$RESULT_FILE"
cat "$result_file" >> "$RESULT_FILE"
fi
done
cat >> "$RESULT_FILE" << EOF
}
}
EOF
print_success "Combined results saved to: $RESULT_FILE"
echo ""
}
generate_summary() {
print_header "Benchmark Summary"
echo "Results saved to: $RESULT_FILE"
echo ""
echo "Individual results:"
ls -lh "$RESULTS_DIR"/*_${TIMESTAMP}.json 2>/dev/null || true
echo ""
print_info "To generate HTML report, run:"
echo " python scripts/generate_report.py --input $RESULT_FILE --output report.html"
echo ""
}
###############################################################################
# Main
###############################################################################
# Parse command line arguments
while [[ $# -gt 0 ]]; do
case $1 in
-t|--tracker)
TRACKER_SERVER="$2"
shift 2
;;
-T|--threads)
THREADS="$2"
shift 2
;;
-o|--output)
RESULT_FILE="$2"
shift 2
;;
-v|--verbose)
VERBOSE=1
shift
;;
-h|--help)
usage
exit 0
;;
*)
print_error "Unknown option: $1"
usage
exit 1
;;
esac
done
# Main execution
print_header "FastDFS Benchmark Suite"
echo "Timestamp: $(date)"
echo "Tracker: $TRACKER_SERVER"
echo "Threads: $THREADS"
echo ""
check_prerequisites
# Run benchmarks
run_benchmark "upload" "benchmark_upload" \
--tracker "$TRACKER_SERVER" \
--threads $THREADS \
--files 1000 \
--size 1048576
run_benchmark "download" "benchmark_download" \
--tracker "$TRACKER_SERVER" \
--threads $THREADS \
--iterations 1000 \
--prepare 100
run_benchmark "concurrent" "benchmark_concurrent" \
--tracker "$TRACKER_SERVER" \
--users $THREADS \
--duration 60 \
--mix "50:45:5"
run_benchmark "small_files" "benchmark_small_files" \
--tracker "$TRACKER_SERVER" \
--threads $THREADS \
--count 10000 \
--min-size 1024 \
--max-size 102400
run_benchmark "large_files" "benchmark_large_files" \
--tracker "$TRACKER_SERVER" \
--threads 5 \
--count 10 \
--min-size 104857600 \
--max-size 524288000
run_benchmark "metadata" "benchmark_metadata" \
--tracker "$TRACKER_SERVER" \
--threads $THREADS \
--operations 10000 \
--mix "70:20:10"
# Combine results
combine_results
# Generate summary
generate_summary
print_success "All benchmarks completed successfully!"
exit 0
================================================
FILE: cli/Makefile.in
================================================
.SUFFIXES: .c .o
COMPILE = $(CC) $(CFLAGS)
INC_PATH = -I../common -I../tracker -I../client -I/usr/include/fastcommon
LIB_PATH = $(LIBS) -lfastcommon -lserverframe -lfdfsclient
TARGET_PATH = $(TARGET_PREFIX)/bin
FDFS_CLI_OBJS = ../common/fdfs_global.o ../common/fdfs_http_shared.o \
../common/mime_file_parser.o ../tracker/tracker_proto.o \
../tracker/fdfs_shared_func.o ../tracker/fdfs_server_id_func.o \
../storage/trunk_mgr/trunk_shared.o \
../client/tracker_client.o ../client/client_func.o \
../client/client_global.o ../client/storage_client.o
ALL_PRGS = fdfs_cli
all: $(ALL_PRGS)
fdfs_cli: fdfs_cli.c $(FDFS_CLI_OBJS)
$(COMPILE) -o $@ $< $(FDFS_CLI_OBJS) $(LIB_PATH) $(INC_PATH)
.c.o:
$(COMPILE) -c -o $@ $< $(INC_PATH)
install:
mkdir -p $(TARGET_PATH)
cp -f $(ALL_PRGS) $(TARGET_PATH)
clean:
rm -f $(ALL_PRGS) *.o
================================================
FILE: cli/README.md
================================================
# FastDFS Modern CLI Tool
A modern, feature-rich command-line interface for FastDFS with enhanced usability and productivity features.
## Features
### ⭐ Core Capabilities
- **Upload**: Upload files to FastDFS storage
- **Download**: Download files from FastDFS storage
- **Delete**: Remove files from storage
- **Info**: Get detailed file information
- **Batch Operations**: Process multiple files from a list
- **Interactive Mode**: REPL-style interface for multiple operations
### 🎨 Modern UX Features
- **Colored Output**: Easy-to-read color-coded messages
- **Progress Bars**: Visual feedback for long operations
- **JSON Output**: Machine-readable output for automation
- **Human-Readable Sizes**: Automatic file size formatting (B, KB, MB, GB, TB)
- **Timestamp Formatting**: Human-friendly date/time display
## Installation
The CLI tool is built automatically when you build FastDFS:
```bash
cd fastdfs
./make.sh
./make.sh install
```
The `fdfs_cli` binary will be installed to `/usr/bin` (or your configured `TARGET_PREFIX/bin`).
## Usage
### Basic Syntax
```bash
fdfs_cli [options] [args...]
```
### Options
| Option | Description |
|--------|-------------|
| `-c ` | Configuration file path (required) |
| `-j` | Enable JSON output format |
| `-n` | Disable colored output |
| `-v` | Enable verbose mode |
| `-p ` | Specify storage path index |
| `-h` | Show help message |
### Commands
#### Upload a File
```bash
# Upload to default group
fdfs_cli -c /etc/fdfs/client.conf upload /path/to/file.jpg
# Upload to specific group
fdfs_cli -c /etc/fdfs/client.conf upload /path/to/file.jpg group1
# With JSON output
fdfs_cli -c /etc/fdfs/client.conf -j upload /path/to/file.jpg
```
**Output:**
```
Uploading: /path/to/file.jpg (2.45 MB)
Progress [==================================================] 100%
✓ Upload successful!
File ID: group1/M00/00/00/wKgBaGFxxx.jpg
```
#### Download a File
```bash
# Download with auto-generated filename
fdfs_cli -c /etc/fdfs/client.conf download group1/M00/00/00/wKgBaGFxxx.jpg
# Download to specific location
fdfs_cli -c /etc/fdfs/client.conf download group1/M00/00/00/wKgBaGFxxx.jpg /tmp/myfile.jpg
```
**Output:**
```
Downloading: group1/M00/00/00/wKgBaGFxxx.jpg
Progress [==================================================] 100%
✓ Download successful!
Saved to: /tmp/myfile.jpg (2.45 MB)
```
#### Delete a File
```bash
fdfs_cli -c /etc/fdfs/client.conf delete group1/M00/00/00/wKgBaGFxxx.jpg
```
**Output:**
```
✓ File deleted: group1/M00/00/00/wKgBaGFxxx.jpg
```
#### Get File Information
```bash
fdfs_cli -c /etc/fdfs/client.conf info group1/M00/00/00/wKgBaGFxxx.jpg
```
**Output:**
```
File Information
================
File ID: group1/M00/00/00/wKgBaGFxxx.jpg
Size: 2.45 MB (2568192 bytes)
Created: 2025-11-19 22:30:45
CRC32: 0x12345678
Source IP: 192.168.1.100
```
#### Batch Operations
Create a file list (`files.txt`):
```
/path/to/file1.jpg
/path/to/file2.png
/path/to/file3.pdf
# Comments are supported
/path/to/file4.doc
```
Run batch upload:
```bash
fdfs_cli -c /etc/fdfs/client.conf batch upload files.txt
```
**Output:**
```
Batch upload: 4 files
Batch [==================================================] 100%
Summary: Success=4 Failed=0 Total=4
```
Batch operations support:
- `batch upload ` - Upload multiple files
- `batch download ` - Download multiple files (list contains file IDs)
- `batch delete ` - Delete multiple files (list contains file IDs)
#### Interactive Mode
```bash
fdfs_cli -c /etc/fdfs/client.conf interactive
```
**Interactive Session:**
```
FastDFS Interactive CLI
Type 'help' for commands, 'exit' to quit
fdfs> help
Commands: upload [group] | download [dest] | delete | info | batch | exit
fdfs> upload test.jpg
Uploading: test.jpg (1.23 MB)
Progress [==================================================] 100%
✓ Upload successful!
File ID: group1/M00/00/00/wKgBaGFxxx.jpg
fdfs> info group1/M00/00/00/wKgBaGFxxx.jpg
File Information
================
File ID: group1/M00/00/00/wKgBaGFxxx.jpg
Size: 1.23 MB (1290240 bytes)
Created: 2025-11-19 22:35:12
CRC32: 0xABCDEF01
Source IP: 192.168.1.100
fdfs> exit
Goodbye!
```
## JSON Output Format
Enable JSON output with the `-j` flag for easy integration with scripts and automation tools.
### Upload Response
```json
{
"operation": "upload",
"success": true,
"file_id": "group1/M00/00/00/wKgBaGFxxx.jpg"
}
```
### Download Response
```json
{
"operation": "download",
"success": true,
"file_id": "group1/M00/00/00/wKgBaGFxxx.jpg",
"local": "/tmp/myfile.jpg",
"size": 2568192
}
```
### Info Response
```json
{
"operation": "info",
"success": true,
"file_id": "group1/M00/00/00/wKgBaGFxxx.jpg",
"size": 2568192,
"timestamp": 1700432445,
"crc32": 305441401,
"source_ip": "192.168.1.100"
}
```
### Error Response
```json
{
"operation": "upload",
"success": false,
"error_code": 2,
"error": "No such file or directory"
}
```
### Batch Response
```json
{
"operation": "batch_upload",
"total": 10,
"success": 9,
"failed": 1
}
```
## Examples
### Automation Script
```bash
#!/bin/bash
# Upload with error handling
result=$(fdfs_cli -c /etc/fdfs/client.conf -j upload photo.jpg)
if echo "$result" | grep -q '"success":true'; then
file_id=$(echo "$result" | grep -o '"file_id":"[^"]*"' | cut -d'"' -f4)
echo "Uploaded successfully: $file_id"
else
echo "Upload failed"
exit 1
fi
```
### Batch Processing
```bash
# Generate file list
find /photos -name "*.jpg" > upload_list.txt
# Batch upload
fdfs_cli -c /etc/fdfs/client.conf batch upload upload_list.txt
# With JSON output for logging
fdfs_cli -c /etc/fdfs/client.conf -j batch upload upload_list.txt >> upload_log.json
```
### Pipeline Integration
```bash
# Upload and immediately get info
file_id=$(fdfs_cli -c /etc/fdfs/client.conf upload test.jpg | tail -1)
fdfs_cli -c /etc/fdfs/client.conf info "$file_id"
```
## Configuration
The CLI tool uses the standard FastDFS client configuration file. Example `/etc/fdfs/client.conf`:
```ini
connect_timeout = 30
network_timeout = 60
base_path = /tmp
tracker_server = 192.168.1.100:22122
tracker_server = 192.168.1.101:22122
```
## Tips & Best Practices
1. **Use JSON output for scripts**: The `-j` flag provides consistent, parseable output
2. **Batch operations for efficiency**: Process multiple files in one command
3. **Interactive mode for exploration**: Great for testing and learning
4. **Disable colors in scripts**: Use `-n` flag when piping output
5. **Store file IDs**: Keep track of uploaded file IDs for later retrieval
## Troubleshooting
### Connection Errors
```
Error: Tracker connection failed
```
- Check if tracker server is running
- Verify `tracker_server` in config file
- Check network connectivity
### File Not Found
```
Error: File not found: /path/to/file
```
- Verify file path is correct
- Check file permissions
### Configuration Issues
```
Error: Configuration file required (-c option)
```
- Always specify config file with `-c` option
- Verify config file exists and is readable
## Performance Notes
- **Batch operations** are more efficient than individual commands in loops
- **Progress bars** add minimal overhead and can be disabled with `-n` for maximum performance
- **JSON output** has slightly less overhead than formatted output
## Comparison with Original Tools
| Feature | Original Tools | Modern CLI |
|---------|---------------|------------|
| Upload | `fdfs_upload_file` | `fdfs_cli upload` |
| Download | `fdfs_download_file` | `fdfs_cli download` |
| Delete | `fdfs_delete_file` | `fdfs_cli delete` |
| Info | `fdfs_file_info` | `fdfs_cli info` |
| Batch | ❌ | ✅ |
| Interactive | ❌ | ✅ |
| Progress Bars | ❌ | ✅ |
| Colored Output | ❌ | ✅ |
| JSON Output | ❌ | ✅ |
| Single Binary | ❌ | ✅ |
## License
Copyright (C) 2008 Happy Fish / YuQing
FastDFS may be copied only under the terms of the GNU General Public License V3.
## Contributing
Contributions are welcome! Please see the main FastDFS repository for contribution guidelines.
================================================
FILE: cli/fdfs_cli.c
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
*
* Modern CLI Enhancement for FastDFS
* Features: Interactive mode, progress bars, colored output, JSON format,
* batch operations, search/filter capabilities
**/
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include "fdfs_client.h"
#include "fastcommon/logger.h"
#include "fastcommon/shared_func.h"
/* ANSI Color Codes */
#define COLOR_RESET "\033[0m"
#define COLOR_RED "\033[31m"
#define COLOR_GREEN "\033[32m"
#define COLOR_YELLOW "\033[33m"
#define COLOR_BLUE "\033[34m"
#define COLOR_CYAN "\033[36m"
#define COLOR_BOLD "\033[1m"
#define MAX_LINE_LENGTH 4096
#define PROGRESS_BAR_WIDTH 50
typedef struct {
char config_file[256];
int color_enabled;
int json_output;
int verbose;
int store_path_index;
} CLIConfig;
static CLIConfig g_cli = {
.color_enabled = 1,
.json_output = 0,
.verbose = 0,
.store_path_index = -1
};
/* Function prototypes */
static void print_colored(const char *color, const char *fmt, ...);
static void print_progress(int64_t cur, int64_t total, const char *label);
static char* fmt_size(int64_t size, char *buf, int len);
static char* fmt_time(time_t t, char *buf, int len);
static void print_json(const char *op, int res, const char *fid, const char *err);
static int cmd_upload(int argc, char *argv[]);
static int cmd_download(int argc, char *argv[]);
static int cmd_delete(int argc, char *argv[]);
static int cmd_info(int argc, char *argv[]);
static int cmd_batch(int argc, char *argv[]);
static int cmd_interactive(void);
static void usage(char *argv[]);
/* Print colored output */
static void print_colored(const char *color, const char *fmt, ...) {
va_list args;
if (g_cli.color_enabled && !g_cli.json_output) {
printf("%s", color);
}
va_start(args, fmt);
vprintf(fmt, args);
va_end(args);
if (g_cli.color_enabled && !g_cli.json_output) {
printf("%s", COLOR_RESET);
}
}
/* Print progress bar */
static void print_progress(int64_t cur, int64_t total, const char *label) {
if (g_cli.json_output || !g_cli.color_enabled) {
return;
}
int pct = (int)((cur * 100) / total);
int filled = (cur * PROGRESS_BAR_WIDTH) / total;
printf("\r%s [", label);
for (int i = 0; i < PROGRESS_BAR_WIDTH; i++) {
if (i < filled) {
printf("=");
} else if (i == filled) {
printf(">");
} else {
printf(" ");
}
}
printf("] %d%%", pct);
fflush(stdout);
if (cur >= total) {
printf("\n");
}
}
/* Format file size */
static char* fmt_size(int64_t size, char *buf, int len) {
const char *units[] = {"B", "KB", "MB", "GB", "TB"};
int idx = 0;
double s = (double)size;
while (s >= 1024.0 && idx < 4) {
s /= 1024.0;
idx++;
}
snprintf(buf, len, "%.2f %s", s, units[idx]);
return buf;
}
/* Format timestamp */
static char* fmt_time(time_t t, char *buf, int len) {
struct tm *tm = localtime(&t);
strftime(buf, len, "%Y-%m-%d %H:%M:%S", tm);
return buf;
}
/* Print JSON result */
static void print_json(const char *op, int res, const char *fid, const char *err) {
printf("{\"operation\":\"%s\",\"success\":%s", op, res == 0 ? "true" : "false");
if (res == 0 && fid != NULL) {
printf(",\"file_id\":\"%s\"", fid);
}
if (res != 0 && err != NULL) {
printf(",\"error_code\":%d,\"error\":\"%s\"", res, err);
}
printf("}\n");
}
/* Command: Upload file */
static int cmd_upload(int argc, char *argv[]) {
if (argc < 1) {
print_colored(COLOR_RED, "Error: Missing filename\n");
return 1;
}
const char *local = argv[0];
char *group = (argc >= 2) ? argv[1] : "";
char fid[128] = {0}, remote[128] = {0};
ConnectionInfo *tracker, storage;
int result;
struct stat st;
if (stat(local, &st) != 0) {
if (g_cli.json_output) {
print_json("upload", errno, NULL, strerror(errno));
} else {
print_colored(COLOR_RED, "Error: File not found: %s\n", local);
}
return errno;
}
if ((result = fdfs_client_init(g_cli.config_file)) != 0) {
if (g_cli.json_output) {
print_json("upload", result, NULL, STRERROR(result));
} else {
print_colored(COLOR_RED, "Error: Init failed: %s\n", STRERROR(result));
}
return result;
}
tracker = tracker_get_connection();
if (tracker == NULL) {
result = errno != 0 ? errno : ECONNREFUSED;
if (g_cli.json_output) {
print_json("upload", result, NULL, "Tracker connection failed");
} else {
print_colored(COLOR_RED, "Error: Tracker connection failed\n");
}
fdfs_client_destroy();
return result;
}
if ((result = tracker_query_storage_store(tracker, &storage, group, &g_cli.store_path_index)) != 0) {
if (g_cli.json_output) {
print_json("upload", result, NULL, STRERROR(result));
} else {
print_colored(COLOR_RED, "Error: Query storage failed: %s\n", STRERROR(result));
}
tracker_close_connection_ex(tracker, true);
fdfs_client_destroy();
return result;
}
if (!g_cli.json_output) {
char sz[32];
print_colored(COLOR_CYAN, "Uploading: %s (%s)\n", local, fmt_size(st.st_size, sz, sizeof(sz)));
print_progress(0, 100, "Progress");
}
result = storage_upload_by_filename1(tracker, &storage, g_cli.store_path_index,
local, NULL, NULL, 0, group, remote);
if (result == 0) {
fdfs_combine_file_id(group, remote, fid);
if (g_cli.json_output) {
print_json("upload", 0, fid, NULL);
} else {
print_progress(100, 100, "Progress");
print_colored(COLOR_GREEN, "✓ Upload successful!\n");
print_colored(COLOR_BOLD, "File ID: %s\n", fid);
}
} else {
if (g_cli.json_output) {
print_json("upload", result, NULL, STRERROR(result));
} else {
print_colored(COLOR_RED, "✗ Upload failed: %s\n", STRERROR(result));
}
}
tracker_close_connection_ex(tracker, true);
fdfs_client_destroy();
return result;
}
/* Command: Download file */
static int cmd_download(int argc, char *argv[]) {
if (argc < 1) {
print_colored(COLOR_RED, "Error: Missing file ID\n");
return 1;
}
const char *fid = argv[0];
const char *local = (argc >= 2) ? argv[1] : NULL;
ConnectionInfo *tracker;
int result;
int64_t size = 0;
if ((result = fdfs_client_init(g_cli.config_file)) != 0) {
if (g_cli.json_output) {
print_json("download", result, NULL, STRERROR(result));
} else {
print_colored(COLOR_RED, "Error: Init failed: %s\n", STRERROR(result));
}
return result;
}
tracker = tracker_get_connection();
if (tracker == NULL) {
result = errno != 0 ? errno : ECONNREFUSED;
if (g_cli.json_output) {
print_json("download", result, NULL, "Tracker connection failed");
} else {
print_colored(COLOR_RED, "Error: Tracker connection failed\n");
}
fdfs_client_destroy();
return result;
}
if (!g_cli.json_output) {
print_colored(COLOR_CYAN, "Downloading: %s\n", fid);
print_progress(0, 100, "Progress");
}
result = storage_do_download_file1_ex(tracker, NULL, FDFS_DOWNLOAD_TO_FILE, fid,
0, 0, (char **)&local, NULL, &size);
if (result == 0) {
if (g_cli.json_output) {
printf("{\"operation\":\"download\",\"success\":true,\"file_id\":\"%s\",\"local\":\"%s\",\"size\":%lld}\n",
fid, local, (long long)size);
} else {
char sz[32];
print_progress(100, 100, "Progress");
print_colored(COLOR_GREEN, "✓ Download successful!\n");
print_colored(COLOR_BOLD, "Saved to: %s (%s)\n", local, fmt_size(size, sz, sizeof(sz)));
}
} else {
if (g_cli.json_output) {
print_json("download", result, NULL, STRERROR(result));
} else {
print_colored(COLOR_RED, "✗ Download failed: %s\n", STRERROR(result));
}
}
tracker_close_connection_ex(tracker, true);
fdfs_client_destroy();
return result;
}
/* Command: Delete file */
static int cmd_delete(int argc, char *argv[]) {
if (argc < 1) {
print_colored(COLOR_RED, "Error: Missing file ID\n");
return 1;
}
const char *fid = argv[0];
char group[FDFS_GROUP_NAME_MAX_LEN + 1];
char *fname = strchr(fid, FDFS_FILE_ID_SEPERATOR);
ConnectionInfo *tracker, storage;
int result;
if (fname == NULL) {
if (g_cli.json_output) {
print_json("delete", EINVAL, NULL, "Invalid file ID");
} else {
print_colored(COLOR_RED, "Error: Invalid file ID format\n");
}
return EINVAL;
}
snprintf(group, sizeof(group), "%.*s", (int)(fname - fid), fid);
fname++;
if ((result = fdfs_client_init(g_cli.config_file)) != 0) {
if (g_cli.json_output) {
print_json("delete", result, NULL, STRERROR(result));
} else {
print_colored(COLOR_RED, "Error: Init failed: %s\n", STRERROR(result));
}
return result;
}
tracker = tracker_get_connection();
if (tracker == NULL) {
result = errno != 0 ? errno : ECONNREFUSED;
if (g_cli.json_output) {
print_json("delete", result, NULL, "Tracker connection failed");
} else {
print_colored(COLOR_RED, "Error: Tracker connection failed\n");
}
fdfs_client_destroy();
return result;
}
if ((result = tracker_query_storage_fetch(tracker, &storage, group, fname)) != 0) {
if (g_cli.json_output) {
print_json("delete", result, NULL, STRERROR(result));
} else {
print_colored(COLOR_RED, "Error: Query storage failed: %s\n", STRERROR(result));
}
tracker_close_connection_ex(tracker, true);
fdfs_client_destroy();
return result;
}
result = storage_delete_file(tracker, &storage, group, fname);
if (result == 0) {
if (g_cli.json_output) {
print_json("delete", 0, fid, NULL);
} else {
print_colored(COLOR_GREEN, "✓ File deleted: %s\n", fid);
}
} else {
if (g_cli.json_output) {
print_json("delete", result, NULL, STRERROR(result));
} else {
print_colored(COLOR_RED, "✗ Delete failed: %s\n", STRERROR(result));
}
}
tracker_close_connection_ex(tracker, true);
fdfs_client_destroy();
return result;
}
/* Command: Get file info */
static int cmd_info(int argc, char *argv[]) {
if (argc < 1) {
print_colored(COLOR_RED, "Error: Missing file ID\n");
return 1;
}
const char *fid = argv[0];
char group[FDFS_GROUP_NAME_MAX_LEN + 1];
char *fname = strchr(fid, FDFS_FILE_ID_SEPERATOR);
FDFSFileInfo info;
int result;
if (fname == NULL) {
if (g_cli.json_output) {
print_json("info", EINVAL, NULL, "Invalid file ID");
} else {
print_colored(COLOR_RED, "Error: Invalid file ID format\n");
}
return EINVAL;
}
snprintf(group, sizeof(group), "%.*s", (int)(fname - fid), fid);
fname++;
if ((result = fdfs_client_init(g_cli.config_file)) != 0) {
if (g_cli.json_output) {
print_json("info", result, NULL, STRERROR(result));
} else {
print_colored(COLOR_RED, "Error: Init failed: %s\n", STRERROR(result));
}
return result;
}
result = fdfs_get_file_info_ex(group, fname, true, &info, 0);
if (result == 0) {
char sz[32], tm[32];
if (g_cli.json_output) {
printf("{\"operation\":\"info\",\"success\":true,\"file_id\":\"%s\",\"size\":%lld,\"timestamp\":%ld,\"crc32\":%d,\"source_ip\":\"%s\"}\n",
fid, (long long)info.file_size, (long)info.create_timestamp, info.crc32, info.source_ip_addr);
} else {
print_colored(COLOR_BOLD COLOR_CYAN, "File Information\n");
print_colored(COLOR_BOLD COLOR_CYAN, "================\n");
printf("File ID: %s\n", fid);
printf("Size: %s (%lld bytes)\n", fmt_size(info.file_size, sz, sizeof(sz)), (long long)info.file_size);
printf("Created: %s\n", fmt_time(info.create_timestamp, tm, sizeof(tm)));
printf("CRC32: 0x%08X\n", info.crc32);
printf("Source IP: %s\n", info.source_ip_addr);
}
} else {
if (g_cli.json_output) {
print_json("info", result, NULL, STRERROR(result));
} else {
print_colored(COLOR_RED, "✗ Failed to get info: %s\n", STRERROR(result));
}
}
fdfs_client_destroy();
return result;
}
/* Command: Batch operations */
static int cmd_batch(int argc, char *argv[]) {
if (argc < 2) {
print_colored(COLOR_RED, "Error: Usage: batch \n");
return 1;
}
const char *op = argv[0];
const char *list = argv[1];
FILE *fp = fopen(list, "r");
char line[MAX_LINE_LENGTH];
int total = 0, success = 0, failed = 0;
if (fp == NULL) {
print_colored(COLOR_RED, "Error: Cannot open: %s\n", list);
return errno;
}
/* Count total files */
while (fgets(line, sizeof(line), fp) != NULL) {
if (line[0] != '\0' && line[0] != '\n' && line[0] != '#') {
total++;
}
}
rewind(fp);
if (!g_cli.json_output) {
print_colored(COLOR_CYAN, "Batch %s: %d files\n", op, total);
}
int cur = 0;
while (fgets(line, sizeof(line), fp) != NULL) {
line[strcspn(line, "\r\n")] = 0;
if (line[0] == '\0' || line[0] == '\n' || line[0] == '#') {
continue;
}
cur++;
char *args[2] = {line, NULL};
int res = -1;
if (strcmp(op, "upload") == 0) {
res = cmd_upload(1, args);
} else if (strcmp(op, "download") == 0) {
res = cmd_download(1, args);
} else if (strcmp(op, "delete") == 0) {
res = cmd_delete(1, args);
} else {
print_colored(COLOR_RED, "Error: Unknown operation: %s\n", op);
fclose(fp);
return 1;
}
if (res == 0) {
success++;
} else {
failed++;
}
if (!g_cli.json_output) {
print_progress(cur, total, "Batch");
}
}
fclose(fp);
if (g_cli.json_output) {
printf("{\"operation\":\"batch_%s\",\"total\":%d,\"success\":%d,\"failed\":%d}\n",
op, total, success, failed);
} else {
print_colored(COLOR_BOLD, "\nSummary: ");
print_colored(COLOR_GREEN, "Success=%d ", success);
print_colored(COLOR_RED, "Failed=%d ", failed);
print_colored(COLOR_BOLD, "Total=%d\n", total);
}
return (failed > 0) ? 1 : 0;
}
/* Command: Interactive mode */
static int cmd_interactive(void) {
char line[MAX_LINE_LENGTH];
char *args[32];
int argc;
print_colored(COLOR_BOLD COLOR_CYAN, "FastDFS Interactive CLI\n");
print_colored(COLOR_CYAN, "Type 'help' for commands, 'exit' to quit\n\n");
while (1) {
print_colored(COLOR_GREEN, "fdfs> ");
fflush(stdout);
if (fgets(line, sizeof(line), stdin) == NULL) {
break;
}
line[strcspn(line, "\r\n")] = 0;
if (line[0] == '\0') {
continue;
}
argc = 0;
char *tok = strtok(line, " \t");
while (tok != NULL && argc < 32) {
args[argc++] = tok;
tok = strtok(NULL, " \t");
}
if (argc == 0) {
continue;
}
const char *cmd = args[0];
if (strcmp(cmd, "exit") == 0 || strcmp(cmd, "quit") == 0) {
print_colored(COLOR_CYAN, "Goodbye!\n");
break;
} else if (strcmp(cmd, "help") == 0) {
printf("Commands: upload [group] | download [dest] | delete | info | batch | exit\n");
} else if (strcmp(cmd, "upload") == 0) {
cmd_upload(argc - 1, &args[1]);
} else if (strcmp(cmd, "download") == 0) {
cmd_download(argc - 1, &args[1]);
} else if (strcmp(cmd, "delete") == 0) {
cmd_delete(argc - 1, &args[1]);
} else if (strcmp(cmd, "info") == 0) {
cmd_info(argc - 1, &args[1]);
} else if (strcmp(cmd, "batch") == 0) {
cmd_batch(argc - 1, &args[1]);
} else {
print_colored(COLOR_RED, "Unknown command. Type 'help'\n");
}
printf("\n");
}
return 0;
}
/* Print usage */
static void usage(char *argv[]) {
printf("FastDFS Modern CLI Tool\n\n");
printf("Usage: %s [options] [args...]\n\n", argv[0]);
printf("Options:\n");
printf(" -c Configuration file (required)\n");
printf(" -j JSON output\n");
printf(" -n No colors\n");
printf(" -v Verbose\n");
printf(" -p Storage path index\n");
printf(" -h Help\n\n");
printf("Commands:\n");
printf(" upload [group] Upload file\n");
printf(" download [dest] Download file\n");
printf(" delete Delete file\n");
printf(" info File information\n");
printf(" batch Batch operations\n");
printf(" interactive Interactive mode\n\n");
printf("Examples:\n");
printf(" %s -c /etc/fdfs/client.conf upload test.jpg\n", argv[0]);
printf(" %s -c /etc/fdfs/client.conf -j info group1/M00/00/00/test.jpg\n", argv[0]);
printf(" %s -c /etc/fdfs/client.conf batch upload files.txt\n", argv[0]);
printf(" %s -c /etc/fdfs/client.conf interactive\n", argv[0]);
}
/* Main function */
int main(int argc, char *argv[]) {
int opt;
const char *command = NULL;
log_init();
g_log_context.log_level = LOG_ERR;
ignore_signal_pipe();
while ((opt = getopt(argc, argv, "c:jnvp:h")) != -1) {
switch (opt) {
case 'c':
snprintf(g_cli.config_file, sizeof(g_cli.config_file), "%s", optarg);
break;
case 'j':
g_cli.json_output = 1;
break;
case 'n':
g_cli.color_enabled = 0;
break;
case 'v':
g_cli.verbose = 1;
break;
case 'p':
g_cli.store_path_index = atoi(optarg);
break;
case 'h':
usage(argv);
return 0;
default:
usage(argv);
return 1;
}
}
if (g_cli.config_file[0] == '\0') {
print_colored(COLOR_RED, "Error: Configuration file required (-c option)\n");
usage(argv);
return 1;
}
if (optind >= argc) {
print_colored(COLOR_RED, "Error: Command required\n");
usage(argv);
return 1;
}
command = argv[optind];
int cmd_argc = argc - optind - 1;
char **cmd_argv = &argv[optind + 1];
if (strcmp(command, "upload") == 0) {
return cmd_upload(cmd_argc, cmd_argv);
} else if (strcmp(command, "download") == 0) {
return cmd_download(cmd_argc, cmd_argv);
} else if (strcmp(command, "delete") == 0) {
return cmd_delete(cmd_argc, cmd_argv);
} else if (strcmp(command, "info") == 0) {
return cmd_info(cmd_argc, cmd_argv);
} else if (strcmp(command, "batch") == 0) {
return cmd_batch(cmd_argc, cmd_argv);
} else if (strcmp(command, "interactive") == 0) {
return cmd_interactive();
} else {
print_colored(COLOR_RED, "Error: Unknown command: %s\n", command);
usage(argv);
return 1;
}
}
================================================
FILE: client/Makefile.in
================================================
.SUFFIXES: .c .o .lo
COMPILE = $(CC) $(CFLAGS)
ENABLE_STATIC_LIB = $(ENABLE_STATIC_LIB)
ENABLE_SHARED_LIB = $(ENABLE_SHARED_LIB)
INC_PATH = -I../common -I../tracker -I/usr/include/fastcommon
LIB_PATH = $(LIBS) -lfastcommon -lserverframe
TARGET_PATH = $(TARGET_PREFIX)/bin
TARGET_LIB = $(TARGET_PREFIX)/$(LIB_VERSION)
TARGET_INC = $(TARGET_PREFIX)/include
CONFIG_PATH = $(TARGET_CONF_PATH)
FDFS_STATIC_OBJS = ../common/fdfs_global.o ../common/fdfs_http_shared.o \
../common/mime_file_parser.o ../tracker/tracker_proto.o \
../tracker/fdfs_shared_func.o ../tracker/fdfs_server_id_func.o \
../storage/trunk_mgr/trunk_shared.o \
tracker_client.o client_func.o \
client_global.o storage_client.o
STATIC_OBJS = $(FDFS_STATIC_OBJS)
FDFS_SHARED_OBJS = ../common/fdfs_global.lo ../common/fdfs_http_shared.lo \
../common/mime_file_parser.lo ../tracker/tracker_proto.lo \
../tracker/fdfs_shared_func.lo ../tracker/fdfs_server_id_func.lo \
../storage/trunk_mgr/trunk_shared.lo \
tracker_client.lo client_func.lo \
client_global.lo storage_client.lo
FDFS_HEADER_FILES = ../common/fdfs_define.h ../common/fdfs_global.h \
../common/mime_file_parser.h ../common/fdfs_http_shared.h \
../tracker/tracker_types.h ../tracker/tracker_proto.h \
../tracker/fdfs_shared_func.h ../tracker/fdfs_server_id_func.h \
../storage/trunk_mgr/trunk_shared.h \
tracker_client.h storage_client.h storage_client1.h \
client_func.h client_global.h fdfs_client.h
ALL_OBJS = $(STATIC_OBJS) $(FDFS_SHARED_OBJS)
ALL_PRGS = fdfs_monitor fdfs_test fdfs_test1 fdfs_crc32 fdfs_upload_file \
fdfs_download_file fdfs_delete_file fdfs_file_info \
fdfs_appender_test fdfs_appender_test1 fdfs_append_file \
fdfs_upload_appender fdfs_regenerate_filename
STATIC_LIBS = libfdfsclient.a
SHARED_LIBS = libfdfsclient.so
CLIENT_SHARED_LIBS = libfdfsclient.so
ALL_LIBS = $(STATIC_LIBS) $(SHARED_LIBS)
all: $(ALL_OBJS) $(ALL_PRGS) $(ALL_LIBS)
libfdfsclient.so:
$(COMPILE) -o $@ $< -shared $(FDFS_SHARED_OBJS) $(LIB_PATH)
libfdfsclient.a:
ar rcs $@ $< $(FDFS_STATIC_OBJS)
.o:
$(COMPILE) -o $@ $< $(STATIC_OBJS) $(LIB_PATH) $(INC_PATH)
.c:
$(COMPILE) -o $@ $< $(STATIC_OBJS) $(LIB_PATH) $(INC_PATH)
.c.o:
$(COMPILE) -c -o $@ $< $(INC_PATH)
.c.lo:
$(COMPILE) -c -fPIC -o $@ $< $(INC_PATH)
install:
mkdir -p $(TARGET_PATH)
mkdir -p $(CONFIG_PATH)
mkdir -p $(TARGET_LIB)
mkdir -p $(TARGET_PREFIX)/lib
cp -f $(ALL_PRGS) $(TARGET_PATH)
if [ $(ENABLE_STATIC_LIB) -eq 1 ]; then cp -f $(STATIC_LIBS) $(TARGET_LIB); cp -f $(STATIC_LIBS) $(TARGET_PREFIX)/lib/; fi
if [ $(ENABLE_SHARED_LIB) -eq 1 ]; then cp -f $(CLIENT_SHARED_LIBS) $(TARGET_LIB); cp -f $(CLIENT_SHARED_LIBS) $(TARGET_PREFIX)/lib/; fi
mkdir -p $(TARGET_INC)/fastdfs
cp -f $(FDFS_HEADER_FILES) $(TARGET_INC)/fastdfs
if [ ! -f $(CONFIG_PATH)/client.conf ]; then cp -f ../conf/client.conf $(CONFIG_PATH)/client.conf; fi
clean:
rm -f $(ALL_OBJS) $(ALL_PRGS) $(ALL_LIBS)
================================================
FILE: client/client_func.c
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//client_func.c
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include "fdfs_define.h"
#include "fastcommon/logger.h"
#include "fdfs_global.h"
#include "fastcommon/base64.h"
#include "fastcommon/sockopt.h"
#include "fastcommon/shared_func.h"
#include "fastcommon/ini_file_reader.h"
#include "fastcommon/connection_pool.h"
#include "tracker_types.h"
#include "tracker_proto.h"
#include "client_global.h"
#include "client_func.h"
static int storage_cmp_by_ip_and_port(const void *p1, const void *p2)
{
int res;
res = strcmp(((ConnectionInfo *)p1)->ip_addr,
((ConnectionInfo *)p2)->ip_addr);
if (res != 0)
{
return res;
}
return ((ConnectionInfo *)p1)->port -
((ConnectionInfo *)p2)->port;
}
static int storage_cmp_server_info(const void *p1, const void *p2)
{
TrackerServerInfo *server1;
TrackerServerInfo *server2;
ConnectionInfo *pc1;
ConnectionInfo *pc2;
ConnectionInfo *end1;
int res;
server1 = (TrackerServerInfo *)p1;
server2 = (TrackerServerInfo *)p2;
res = server1->count - server2->count;
if (res != 0)
{
return res;
}
if (server1->count == 1)
{
return storage_cmp_by_ip_and_port(server1->connections + 0,
server2->connections + 0);
}
end1 = server1->connections + server1->count;
for (pc1=server1->connections,pc2=server2->connections; pc1servers+pTrackerGroup->server_count;
pDestServer>pTrackerGroup->servers; pDestServer--)
{
if (storage_cmp_server_info(pInsertedServer, pDestServer-1) > 0)
{
memcpy(pDestServer, pInsertedServer,
sizeof(TrackerServerInfo));
return;
}
memcpy(pDestServer, pDestServer-1, sizeof(TrackerServerInfo));
}
memcpy(pDestServer, pInsertedServer, sizeof(TrackerServerInfo));
}
static int copy_tracker_servers(TrackerServerGroup *pTrackerGroup,
const char *filename, char **ppTrackerServers)
{
char **ppSrc;
char **ppEnd;
TrackerServerInfo destServer;
int result;
memset(&destServer, 0, sizeof(TrackerServerInfo));
fdfs_server_sock_reset(&destServer);
ppEnd = ppTrackerServers + pTrackerGroup->server_count;
pTrackerGroup->server_count = 0;
for (ppSrc=ppTrackerServers; ppSrcservers,
pTrackerGroup->server_count,
sizeof(TrackerServerInfo),
storage_cmp_server_info) == NULL)
{
insert_into_sorted_servers(pTrackerGroup, &destServer);
pTrackerGroup->server_count++;
}
}
/*
{
TrackerServerInfo *pServer;
char formatted_ip[FORMATTED_IP_SIZE];
for (pServer=pTrackerGroup->servers; pServerservers+
pTrackerGroup->server_count; pServer++)
{
format_ip_address(pServer->connections[0].ip_addr, formatted_ip);
printf("server=%s:%u\n", formatted_ip, pServer->connections[0].port);
}
}
*/
return 0;
}
static int fdfs_check_tracker_group(TrackerServerGroup *pTrackerGroup,
const char *conf_filename)
{
int result;
TrackerServerInfo *pServer;
TrackerServerInfo *pEnd;
char error_info[256];
pEnd = pTrackerGroup->servers + pTrackerGroup->server_count;
for (pServer=pTrackerGroup->servers; pServerserver_count=iniGetValues(NULL, "tracker_server",
pIniContext, ppTrackerServers, FDFS_MAX_TRACKERS)) <= 0)
{
logError("file: "__FILE__", line: %d, "
"conf file \"%s\", item \"tracker_server\" not exist",
__LINE__, conf_filename);
return ENOENT;
}
bytes = sizeof(TrackerServerInfo) * pTrackerGroup->server_count;
pTrackerGroup->servers = (TrackerServerInfo *)malloc(bytes);
if (pTrackerGroup->servers == NULL)
{
logError("file: "__FILE__", line: %d, "
"malloc %d bytes fail", __LINE__, bytes);
pTrackerGroup->server_count = 0;
return errno != 0 ? errno : ENOMEM;
}
memset(pTrackerGroup->servers, 0, bytes);
if ((result=copy_tracker_servers(pTrackerGroup, conf_filename,
ppTrackerServers)) != 0)
{
pTrackerGroup->server_count = 0;
free(pTrackerGroup->servers);
pTrackerGroup->servers = NULL;
return result;
}
return fdfs_check_tracker_group(pTrackerGroup, conf_filename);
}
int fdfs_load_tracker_group(TrackerServerGroup *pTrackerGroup,
const char *conf_filename)
{
IniContext iniContext;
int result;
if ((result=iniLoadFromFile(conf_filename, &iniContext)) != 0)
{
logError("file: "__FILE__", line: %d, "
"load conf file \"%s\" fail, ret code: %d",
__LINE__, conf_filename, result);
return result;
}
result = fdfs_load_tracker_group_ex(pTrackerGroup,
conf_filename, &iniContext);
iniFreeContext(&iniContext);
return result;
}
static int fdfs_get_params_from_tracker(bool *use_storage_id)
{
IniContext iniContext;
int result;
bool continue_flag;
continue_flag = false;
if ((result=fdfs_get_ini_context_from_tracker(&g_tracker_group,
&iniContext, &continue_flag)) != 0)
{
return result;
}
*use_storage_id = iniGetBoolValue(NULL, "use_storage_id",
&iniContext, false);
iniFreeContext(&iniContext);
if (*use_storage_id)
{
result = fdfs_get_storage_ids_from_tracker_group(
&g_tracker_group);
}
return result;
}
static int fdfs_client_do_init_ex(TrackerServerGroup *pTrackerGroup, \
const char *conf_filename, IniContext *iniContext)
{
char *pBasePath;
int result;
bool use_storage_id;
bool load_fdfs_parameters_from_tracker;
pBasePath = iniGetStrValue(NULL, "base_path", iniContext);
if (pBasePath == NULL)
{
strcpy(SF_G_BASE_PATH_STR, "/tmp");
}
else
{
fc_safe_strcpy(SF_G_BASE_PATH_STR, pBasePath);
chopPath(SF_G_BASE_PATH_STR);
if (!fileExists(SF_G_BASE_PATH_STR))
{
logError("file: "__FILE__", line: %d, " \
"\"%s\" can't be accessed, error info: %s", \
__LINE__, SF_G_BASE_PATH_STR, STRERROR(errno));
return errno != 0 ? errno : ENOENT;
}
if (!isDir(SF_G_BASE_PATH_STR))
{
logError("file: "__FILE__", line: %d, " \
"\"%s\" is not a directory!", \
__LINE__, SF_G_BASE_PATH_STR);
return ENOTDIR;
}
}
SF_G_BASE_PATH_LEN = strlen(SF_G_BASE_PATH_STR);
SF_G_CONNECT_TIMEOUT = iniGetIntValue(NULL, "connect_timeout", \
iniContext, DEFAULT_CONNECT_TIMEOUT);
if (SF_G_CONNECT_TIMEOUT <= 0)
{
SF_G_CONNECT_TIMEOUT = DEFAULT_CONNECT_TIMEOUT;
}
SF_G_NETWORK_TIMEOUT = iniGetIntValue(NULL, "network_timeout", \
iniContext, DEFAULT_NETWORK_TIMEOUT);
if (SF_G_NETWORK_TIMEOUT <= 0)
{
SF_G_NETWORK_TIMEOUT = DEFAULT_NETWORK_TIMEOUT;
}
if ((result=fdfs_load_tracker_group_ex(pTrackerGroup, \
conf_filename, iniContext)) != 0)
{
return result;
}
g_anti_steal_token = iniGetBoolValue(NULL, \
"http.anti_steal.check_token", \
iniContext, false);
if (g_anti_steal_token)
{
char *anti_steal_secret_key;
anti_steal_secret_key = iniGetStrValue(NULL, \
"http.anti_steal.secret_key", \
iniContext);
if (anti_steal_secret_key == NULL || \
*anti_steal_secret_key == '\0')
{
logError("file: "__FILE__", line: %d, " \
"param \"http.anti_steal.secret_key\""\
" not exist or is empty", __LINE__);
return EINVAL;
}
buffer_strcpy(&g_anti_steal_secret_key, anti_steal_secret_key);
}
if ((result=fdfs_connection_pool_init(conf_filename, iniContext)) != 0)
{
return result;
}
load_fdfs_parameters_from_tracker = iniGetBoolValue(NULL,
"load_fdfs_parameters_from_tracker",
iniContext, false);
if (load_fdfs_parameters_from_tracker)
{
if ((result=fdfs_get_params_from_tracker(&use_storage_id)) != 0)
{
return result;
}
}
else
{
use_storage_id = iniGetBoolValue(NULL, "use_storage_id",
iniContext, false);
if (use_storage_id)
{
if ((result=fdfs_load_storage_ids_from_file(
conf_filename, iniContext)) != 0)
{
return result;
}
}
}
if (use_storage_id)
{
FDFSStorageIdInfo *idInfo;
FDFSStorageIdInfo *end;
char *connect_first_by;
end = g_storage_ids_by_id.ids + g_storage_ids_by_id.count;
for (idInfo=g_storage_ids_by_id.ids; idInfoip_addrs.count > 1)
{
g_multi_storage_ips = true;
break;
}
}
if (g_multi_storage_ips)
{
connect_first_by = iniGetStrValue(NULL,
"connect_first_by", iniContext);
if (connect_first_by != NULL && strncasecmp(connect_first_by,
"last", 4) == 0)
{
g_connect_first_by = fdfs_connect_first_by_last_connected;
}
}
}
#ifdef DEBUG_FLAG
logDebug("base_path=%s, "
"connect_timeout=%d, "
"network_timeout=%d, "
"tracker_server_count=%d, "
"anti_steal_token=%d, "
"anti_steal_secret_key length=%d, "
"use_connection_pool=%d, "
"g_connection_pool_max_idle_time=%ds, "
"use_storage_id=%d, connect_first_by=%s, "
"storage server id count: %d, "
"multi storage ips: %d\n",
SF_G_BASE_PATH_STR, SF_G_CONNECT_TIMEOUT,
SF_G_NETWORK_TIMEOUT, pTrackerGroup->server_count,
g_anti_steal_token, g_anti_steal_secret_key.length,
g_use_connection_pool, g_connection_pool_max_idle_time,
use_storage_id, g_connect_first_by == fdfs_connect_first_by_tracker ?
"tracker" : "last-connected", g_storage_ids_by_id.count,
g_multi_storage_ips);
#endif
return 0;
}
int fdfs_client_init_from_buffer_ex(TrackerServerGroup *pTrackerGroup, \
const char *buffer)
{
IniContext iniContext;
char *new_buff;
int result;
new_buff = strdup(buffer);
if (new_buff == NULL)
{
logError("file: "__FILE__", line: %d, " \
"strdup %d bytes fail", __LINE__, (int)strlen(buffer));
return ENOMEM;
}
result = iniLoadFromBuffer(new_buff, &iniContext);
free(new_buff);
if (result != 0)
{
logError("file: "__FILE__", line: %d, " \
"load parameters from buffer fail, ret code: %d", \
__LINE__, result);
return result;
}
result = fdfs_client_do_init_ex(pTrackerGroup, "buffer", &iniContext);
iniFreeContext(&iniContext);
return result;
}
int fdfs_client_init_ex(TrackerServerGroup *pTrackerGroup, \
const char *conf_filename)
{
IniContext iniContext;
int result;
if ((result=iniLoadFromFile(conf_filename, &iniContext)) != 0)
{
logError("file: "__FILE__", line: %d, " \
"load conf file \"%s\" fail, ret code: %d", \
__LINE__, conf_filename, result);
return result;
}
result = fdfs_client_do_init_ex(pTrackerGroup, conf_filename, \
&iniContext);
iniFreeContext(&iniContext);
return result;
}
int fdfs_copy_tracker_group(TrackerServerGroup *pDestTrackerGroup, \
TrackerServerGroup *pSrcTrackerGroup)
{
int bytes;
TrackerServerInfo *pDestServer;
TrackerServerInfo *pDestServerEnd;
bytes = sizeof(TrackerServerInfo) * pSrcTrackerGroup->server_count;
pDestTrackerGroup->servers = (TrackerServerInfo *)malloc(bytes);
if (pDestTrackerGroup->servers == NULL)
{
logError("file: "__FILE__", line: %d, "
"malloc %d bytes fail", __LINE__, bytes);
return errno != 0 ? errno : ENOMEM;
}
pDestTrackerGroup->server_index = 0;
pDestTrackerGroup->leader_index = 0;
pDestTrackerGroup->server_count = pSrcTrackerGroup->server_count;
memcpy(pDestTrackerGroup->servers, pSrcTrackerGroup->servers, bytes);
pDestServerEnd = pDestTrackerGroup->servers +
pDestTrackerGroup->server_count;
for (pDestServer=pDestTrackerGroup->servers;
pDestServerserver_count != pGroup2->server_count)
{
return false;
}
pEnd1 = pGroup1->servers + pGroup1->server_count;
pServer1 = pGroup1->servers;
pServer2 = pGroup2->servers;
while (pServer1 < pEnd1)
{
if (!fdfs_server_equal(pServer1, pServer2))
{
return false;
}
pServer1++;
pServer2++;
}
return true;
}
void fdfs_client_destroy_ex(TrackerServerGroup *pTrackerGroup)
{
if (pTrackerGroup->servers != NULL)
{
free(pTrackerGroup->servers);
pTrackerGroup->servers = NULL;
pTrackerGroup->server_count = 0;
pTrackerGroup->server_index = 0;
}
}
const char *fdfs_get_file_ext_name_ex(const char *filename,
const bool twoExtName)
{
const char *fileExtName;
const char *p;
const char *pStart;
int extNameLen;
fileExtName = strrchr(filename, '.');
if (fileExtName == NULL)
{
return NULL;
}
extNameLen = strlen(fileExtName + 1);
if (extNameLen > FDFS_FILE_EXT_NAME_MAX_LEN)
{
return NULL;
}
if (strchr(fileExtName + 1, '/') != NULL) //invalid extension name
{
return NULL;
}
if (!twoExtName)
{
return fileExtName + 1;
}
pStart = fileExtName - (FDFS_FILE_EXT_NAME_MAX_LEN - extNameLen) - 1;
if (pStart < filename)
{
pStart = filename;
}
p = fileExtName - 1; //before .
while ((p > pStart) && (*p != '.'))
{
p--;
}
if (p > pStart) //found (extension name have a dot)
{
if (strchr(p + 1, '/') == NULL) //valid extension name
{
return p + 1; //skip .
}
}
return fileExtName + 1; //skip .
}
================================================
FILE: client/client_func.h
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//client_func.h
#include "tracker_types.h"
#include "client_global.h"
#include "fastcommon/ini_file_reader.h"
#ifndef _CLIENT_FUNC_H_
#define _CLIENT_FUNC_H_
#define FDFS_FILE_ID_SEPERATOR '/'
#define FDFS_FILE_ID_SEPERATE_STR "/"
typedef struct {
short file_type;
bool get_from_server;
time_t create_timestamp;
int crc32;
int source_id; //source storage id
int64_t file_size;
char source_ip_addr[IP_ADDRESS_SIZE]; //source storage ip address
} FDFSFileInfo;
#ifdef __cplusplus
extern "C" {
#endif
#define fdfs_client_init(filename) \
fdfs_client_init_ex((&g_tracker_group), filename)
#define fdfs_client_init_from_buffer(buffer) \
fdfs_client_init_from_buffer_ex((&g_tracker_group), buffer)
#define fdfs_client_destroy() \
fdfs_client_destroy_ex((&g_tracker_group))
/**
* client initial from config file
* params:
* pTrackerGroup: tracker group
* conf_filename: client config filename
* return: 0 success, !=0 fail, return the error code
**/
int fdfs_client_init_ex(TrackerServerGroup *pTrackerGroup, \
const char *conf_filename);
/**
* client initial from buffer
* params:
* pTrackerGroup: tracker group
* conf_filename: client config filename
* return: 0 success, !=0 fail, return the error code
**/
int fdfs_client_init_from_buffer_ex(TrackerServerGroup *pTrackerGroup, \
const char *buffer);
/**
* load tracker server group
* params:
* pTrackerGroup: tracker group
* conf_filename: tracker server group config filename
* return: 0 success, !=0 fail, return the error code
**/
int fdfs_load_tracker_group(TrackerServerGroup *pTrackerGroup, \
const char *conf_filename);
/**
* load tracker server group
* params:
* pTrackerGroup: tracker group
* conf_filename: config filename
* items: ini file items
* nItemCount: ini file item count
* return: 0 success, !=0 fail, return the error code
**/
int fdfs_load_tracker_group_ex(TrackerServerGroup *pTrackerGroup, \
const char *conf_filename, IniContext *pIniContext);
/**
* copy tracker server group
* params:
* pDestTrackerGroup: the dest tracker group
* pSrcTrackerGroup: the source tracker group
* return: 0 success, !=0 fail, return the error code
**/
int fdfs_copy_tracker_group(TrackerServerGroup *pDestTrackerGroup, \
TrackerServerGroup *pSrcTrackerGroup);
/**
* client destroy function
* params:
* pTrackerGroup: tracker group
* return: none
**/
void fdfs_client_destroy_ex(TrackerServerGroup *pTrackerGroup);
/**
* tracker group equals
* params:
* pGroup1: tracker group 1
* pGroup2: tracker group 2
* return: true for equals, otherwise false
**/
bool fdfs_tracker_group_equals(TrackerServerGroup *pGroup1, \
TrackerServerGroup *pGroup2);
/**
* get file ext name from filename, extension name do not include dot
* params:
* filename: the filename
* return: file ext name, NULL for no ext name
**/
#define fdfs_get_file_ext_name1(filename) \
fdfs_get_file_ext_name_ex(filename, false)
/**
* get file ext name from filename, extension name maybe include dot
* params:
* filename: the filename
* return: file ext name, NULL for no ext name
**/
#define fdfs_get_file_ext_name2(filename) \
fdfs_get_file_ext_name_ex(filename, true)
#define fdfs_get_file_ext_name(filename) \
fdfs_get_file_ext_name_ex(filename, true)
/**
* get file ext name from filename
* params:
* filename: the filename
* twoExtName: two extension name as the extension name
* return: file ext name, NULL for no ext name
**/
const char *fdfs_get_file_ext_name_ex(const char *filename,
const bool twoExtName);
static inline int fdfs_combine_file_id(const char *group_name,
const char *filename, char *file_id)
{
char *p;
int group_len;
int file_len;
group_len = strlen(group_name);
file_len = strlen(filename);
p = file_id;
memcpy(p, group_name, group_len);
p += group_len;
*p++ = FDFS_FILE_ID_SEPERATOR;
memcpy(p, filename, file_len);
p += file_len;
*p = '\0';
return p - file_id;
}
#ifdef __cplusplus
}
#endif
#endif
================================================
FILE: client/client_global.c
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include
#include
#include "client_global.h"
TrackerServerGroup g_tracker_group = {0, 0, -1, NULL};
bool g_multi_storage_ips = false;
FDFSConnectFirstBy g_connect_first_by = fdfs_connect_first_by_tracker;
bool g_anti_steal_token = false;
BufferInfo g_anti_steal_secret_key = {0};
================================================
FILE: client/client_global.h
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//client_global.h
#ifndef _CLIENT_GLOBAL_H
#define _CLIENT_GLOBAL_H
#include "fastcommon/common_define.h"
#include "tracker_types.h"
#include "fdfs_shared_func.h"
typedef enum {
fdfs_connect_first_by_tracker,
fdfs_connect_first_by_last_connected
} FDFSConnectFirstBy;
#ifdef __cplusplus
extern "C" {
#endif
extern TrackerServerGroup g_tracker_group;
extern bool g_multi_storage_ips;
extern FDFSConnectFirstBy g_connect_first_by;
extern bool g_anti_steal_token;
extern BufferInfo g_anti_steal_secret_key;
#define fdfs_get_tracker_leader_index(leaderIp, leaderPort) \
fdfs_get_tracker_leader_index_ex(&g_tracker_group, \
leaderIp, leaderPort)
#ifdef __cplusplus
}
#endif
#endif
================================================
FILE: client/fdfs_append_file.c
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include
#include
#include
#include
#include
#include
#include
#include "fdfs_client.h"
#include "fastcommon/logger.h"
int main(int argc, char *argv[])
{
char *conf_filename;
char *local_filename;
ConnectionInfo *pTrackerServer;
int result;
char appender_file_id[128];
if (argc < 4)
{
printf("Usage: %s " \
"\n", argv[0]);
return 1;
}
log_init();
g_log_context.log_level = LOG_ERR;
conf_filename = argv[1];
if ((result=fdfs_client_init(conf_filename)) != 0)
{
return result;
}
pTrackerServer = tracker_get_connection();
if (pTrackerServer == NULL)
{
fdfs_client_destroy();
return errno != 0 ? errno : ECONNREFUSED;
}
fc_safe_strcpy(appender_file_id, argv[2]);
local_filename = argv[3];
if ((result=storage_append_by_filename1(pTrackerServer, \
NULL, local_filename, appender_file_id)) != 0)
{
printf("append file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
return result;
}
tracker_close_connection_ex(pTrackerServer, true);
fdfs_client_destroy();
return result;
}
================================================
FILE: client/fdfs_appender_test.c
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include
#include
#include
#include
#include
#include
#include
#include "fdfs_client.h"
#include "fdfs_global.h"
#include "fastcommon/base64.h"
#include "fastcommon/sockopt.h"
#include "fastcommon/logger.h"
#include "fdfs_http_shared.h"
int writeToFileCallback(void *arg, const int64_t file_size, const char *data, \
const int current_size)
{
if (arg == NULL)
{
return EINVAL;
}
if (fwrite(data, current_size, 1, (FILE *)arg) != 1)
{
return errno != 0 ? errno : EIO;
}
return 0;
}
int uploadFileCallback(void *arg, const int64_t file_size, int sock)
{
int64_t total_send_bytes;
char *filename;
if (arg == NULL)
{
return EINVAL;
}
filename = (char *)arg;
return tcpsendfile(sock, filename, file_size, \
SF_G_NETWORK_TIMEOUT, &total_send_bytes);
}
int main(int argc, char *argv[])
{
char *conf_filename;
char *local_filename;
ConnectionInfo *pTrackerServer;
ConnectionInfo *pStorageServer;
int result;
ConnectionInfo storageServer;
char group_name[FDFS_GROUP_NAME_MAX_LEN + 1];
char remote_filename[256];
char appender_filename[256];
FDFSMetaData meta_list[32];
int meta_count;
char token[32 + 1];
char file_id[128];
char file_url[256];
char szDatetime[20];
int url_len;
time_t ts;
int64_t file_offset;
int64_t file_size = 0;
int store_path_index;
FDFSFileInfo file_info;
int upload_type;
const char *file_ext_name;
struct stat stat_buf;
printf("This is FastDFS client test program v%d.%d.%d\n" \
"\nCopyright (C) 2008, Happy Fish / YuQing\n" \
"\nFastDFS may be copied only under the terms of the GNU General\n" \
"Public License V3, which may be found in the FastDFS source kit.\n" \
"Please visit the FastDFS Home Page http://www.fastken.com/ \n" \
"for more detail.\n\n", g_fdfs_version.major, g_fdfs_version.minor,
g_fdfs_version.patch);
if (argc < 3)
{
printf("Usage: %s " \
"[FILE | BUFF | CALLBACK]\n", argv[0]);
return 1;
}
log_init();
//g_log_context.log_level = LOG_DEBUG;
conf_filename = argv[1];
if ((result=fdfs_client_init(conf_filename)) != 0)
{
return result;
}
pTrackerServer = tracker_get_connection();
if (pTrackerServer == NULL)
{
fdfs_client_destroy();
return errno != 0 ? errno : ECONNREFUSED;
}
local_filename = argv[2];
if (argc == 3)
{
upload_type = FDFS_UPLOAD_BY_FILE;
}
else
{
if (strcmp(argv[3], "BUFF") == 0)
{
upload_type = FDFS_UPLOAD_BY_BUFF;
}
else if (strcmp(argv[3], "CALLBACK") == 0)
{
upload_type = FDFS_UPLOAD_BY_CALLBACK;
}
else
{
upload_type = FDFS_UPLOAD_BY_FILE;
}
}
*group_name = '\0';
store_path_index = 0;
if ((result=tracker_query_storage_store(pTrackerServer, \
&storageServer, group_name, &store_path_index)) != 0)
{
fdfs_client_destroy();
printf("tracker_query_storage fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
return result;
}
printf("group_name=%s, ip_addr=%s, port=%d\n", \
group_name, storageServer.ip_addr, \
storageServer.port);
if ((pStorageServer=tracker_make_connection(&storageServer, \
&result)) == NULL)
{
fdfs_client_destroy();
return result;
}
memset(&meta_list, 0, sizeof(meta_list));
meta_count = 0;
strcpy(meta_list[meta_count].name, "ext_name");
strcpy(meta_list[meta_count].value, "jpg");
meta_count++;
strcpy(meta_list[meta_count].name, "width");
strcpy(meta_list[meta_count].value, "160");
meta_count++;
strcpy(meta_list[meta_count].name, "height");
strcpy(meta_list[meta_count].value, "80");
meta_count++;
strcpy(meta_list[meta_count].name, "file_size");
strcpy(meta_list[meta_count].value, "115120");
meta_count++;
file_ext_name = fdfs_get_file_ext_name(local_filename);
if (upload_type == FDFS_UPLOAD_BY_FILE)
{
if (stat(local_filename, &stat_buf) == 0 && \
S_ISREG(stat_buf.st_mode))
{
file_size = stat_buf.st_size;
result = storage_upload_appender_by_filename ( \
pTrackerServer, pStorageServer, \
store_path_index, local_filename, \
file_ext_name, meta_list, meta_count, \
group_name, remote_filename);
}
else
{
result = errno != 0 ? errno : ENOENT;
}
printf("storage_upload_appender_by_filename\n");
}
else if (upload_type == FDFS_UPLOAD_BY_BUFF)
{
char *file_content;
if ((result=getFileContent(local_filename, \
&file_content, &file_size)) == 0)
{
result = storage_upload_appender_by_filebuff( \
pTrackerServer, pStorageServer, \
store_path_index, file_content, \
file_size, file_ext_name, \
meta_list, meta_count, \
group_name, remote_filename);
free(file_content);
}
printf("storage_upload_appender_by_filebuff\n");
}
else
{
if (stat(local_filename, &stat_buf) == 0 && \
S_ISREG(stat_buf.st_mode))
{
file_size = stat_buf.st_size;
result = storage_upload_appender_by_callback( \
pTrackerServer, pStorageServer, \
store_path_index, uploadFileCallback, \
local_filename, file_size, \
file_ext_name, meta_list, meta_count, \
group_name, remote_filename);
}
else
{
result = errno != 0 ? errno : ENOENT;
}
printf("storage_upload_appender_by_callback\n");
}
if (result != 0)
{
printf("upload file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
tracker_close_connection_ex(pStorageServer, true);
fdfs_client_destroy();
return result;
}
sprintf(file_id, "%s/%s", group_name, remote_filename);
url_len = sprintf(file_url, "http://%s/%s",
pTrackerServer->ip_addr, file_id);
if (g_anti_steal_token)
{
ts = time(NULL);
fdfs_http_gen_token(&g_anti_steal_secret_key, file_id, \
ts, token);
sprintf(file_url + url_len, "?token=%s&ts=%d", token, (int)ts);
}
printf("group_name=%s, remote_filename=%s\n", \
group_name, remote_filename);
fdfs_get_file_info(group_name, remote_filename, &file_info);
printf("source ip address: %s\n", file_info.source_ip_addr);
printf("file timestamp=%s\n", formatDatetime(
file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \
szDatetime, sizeof(szDatetime)));
printf("file size=%"PRId64"\n", file_info.file_size);
printf("file crc32=%u\n", file_info.crc32);
printf("file url: %s\n", file_url);
//sleep(90);
strcpy(appender_filename, remote_filename);
if (storage_truncate_file(pTrackerServer, pStorageServer, \
group_name, appender_filename, file_size / 2) != 0)
{
printf("truncate file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
tracker_close_connection_ex(pStorageServer, true);
fdfs_client_destroy();
return result;
}
fdfs_get_file_info(group_name, appender_filename, &file_info);
printf("source ip address: %s\n", file_info.source_ip_addr);
printf("file timestamp=%s\n", formatDatetime(
file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \
szDatetime, sizeof(szDatetime)));
printf("file size=%"PRId64"\n", file_info.file_size);
printf("file crc32=%u\n", file_info.crc32);
printf("file url: %s\n", file_url);
if (file_info.file_size != file_size / 2)
{
fprintf(stderr, "file size: %"PRId64 \
" != %"PRId64"!!!\n", file_info.file_size, file_size / 2);
}
//sleep(100);
if (upload_type == FDFS_UPLOAD_BY_FILE)
{
result = storage_append_by_filename(pTrackerServer, \
pStorageServer, local_filename,
group_name, appender_filename);
printf("storage_append_by_filename\n");
}
else if (upload_type == FDFS_UPLOAD_BY_BUFF)
{
char *file_content;
if ((result=getFileContent(local_filename, \
&file_content, &file_size)) == 0)
{
result = storage_append_by_filebuff(pTrackerServer, \
pStorageServer, file_content, \
file_size, group_name, appender_filename);
free(file_content);
}
printf("storage_append_by_filebuff\n");
}
else
{
if (stat(local_filename, &stat_buf) == 0 && \
S_ISREG(stat_buf.st_mode))
{
file_size = stat_buf.st_size;
result = storage_append_by_callback(pTrackerServer, \
pStorageServer, uploadFileCallback, \
local_filename, file_size, \
group_name, appender_filename);
}
else
{
result = errno != 0 ? errno : ENOENT;
}
printf("storage_append_by_callback\n");
}
if (result != 0)
{
printf("append file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
tracker_close_connection_ex(pStorageServer, true);
fdfs_client_destroy();
return result;
}
printf("append file successfully.\n");
fdfs_get_file_info(group_name, remote_filename, &file_info);
printf("source ip address: %s\n", file_info.source_ip_addr);
printf("file timestamp=%s\n", formatDatetime(
file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \
szDatetime, sizeof(szDatetime)));
printf("file size=%"PRId64"\n", file_info.file_size);
if (file_info.file_size != file_size + file_size / 2)
{
fprintf(stderr, "file size: %"PRId64 \
" != %"PRId64"!!!\n", file_info.file_size, \
file_size + file_size / 2);
}
file_offset = file_info.file_size;
if (upload_type == FDFS_UPLOAD_BY_FILE)
{
result = storage_modify_by_filename(pTrackerServer, \
pStorageServer, local_filename, \
file_offset, group_name, \
appender_filename);
printf("storage_modify_by_filename\n");
}
else if (upload_type == FDFS_UPLOAD_BY_BUFF)
{
char *file_content;
if ((result=getFileContent(local_filename, \
&file_content, &file_size)) == 0)
{
result = storage_modify_by_filebuff(pTrackerServer, \
pStorageServer, file_content, \
file_offset, file_size, group_name, \
appender_filename);
free(file_content);
}
printf("storage_modify_by_filebuff\n");
}
else
{
if (stat(local_filename, &stat_buf) == 0 && \
S_ISREG(stat_buf.st_mode))
{
file_size = stat_buf.st_size;
result = storage_modify_by_callback(pTrackerServer, \
pStorageServer, uploadFileCallback, \
local_filename, file_offset, \
file_size, group_name, appender_filename);
}
else
{
result = errno != 0 ? errno : ENOENT;
}
printf("storage_modify_by_callback\n");
}
if (result != 0)
{
printf("modify file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
tracker_close_connection_ex(pStorageServer, true);
fdfs_client_destroy();
return result;
}
printf("modify file successfully.\n");
fdfs_get_file_info(group_name, remote_filename, &file_info);
printf("source ip address: %s\n", file_info.source_ip_addr);
printf("file timestamp=%s\n", formatDatetime(
file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \
szDatetime, sizeof(szDatetime)));
printf("file size=%"PRId64"\n", file_info.file_size);
if (file_info.file_size != 2 * file_size + file_size / 2)
{
fprintf(stderr, "file size: %"PRId64 \
" != %"PRId64"!!!\n", file_info.file_size, \
2 * file_size + file_size /2);
}
tracker_close_connection_ex(pStorageServer, true);
tracker_close_connection_ex(pTrackerServer, true);
fdfs_client_destroy();
return result;
}
================================================
FILE: client/fdfs_appender_test1.c
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include
#include
#include
#include
#include
#include
#include
#include "fdfs_client.h"
#include "fdfs_global.h"
#include "fastcommon/base64.h"
#include "fastcommon/sockopt.h"
#include "fastcommon/logger.h"
#include "fdfs_http_shared.h"
int writeToFileCallback(void *arg, const int64_t file_size, const char *data, \
const int current_size)
{
if (arg == NULL)
{
return EINVAL;
}
if (fwrite(data, current_size, 1, (FILE *)arg) != 1)
{
return errno != 0 ? errno : EIO;
}
return 0;
}
int uploadFileCallback(void *arg, const int64_t file_size, int sock)
{
int64_t total_send_bytes;
char *filename;
if (arg == NULL)
{
return EINVAL;
}
filename = (char *)arg;
return tcpsendfile(sock, filename, file_size, \
SF_G_NETWORK_TIMEOUT, &total_send_bytes);
}
int main(int argc, char *argv[])
{
char *conf_filename;
char *local_filename;
ConnectionInfo *pTrackerServer;
ConnectionInfo *pStorageServer;
int result;
ConnectionInfo storageServer;
char group_name[FDFS_GROUP_NAME_MAX_LEN + 1];
char file_id[256];
char appender_file_id[256];
FDFSMetaData meta_list[32];
int meta_count;
char token[32 + 1];
char file_url[256];
char szDatetime[20];
int url_len;
time_t ts;
int64_t file_offset;
int64_t file_size = 0;
int store_path_index;
FDFSFileInfo file_info;
int upload_type;
const char *file_ext_name;
struct stat stat_buf;
printf("This is FastDFS client test program v%d.%d.%d\n" \
"\nCopyright (C) 2008, Happy Fish / YuQing\n" \
"\nFastDFS may be copied only under the terms of the GNU General\n" \
"Public License V3, which may be found in the FastDFS source kit.\n" \
"Please visit the FastDFS Home Page http://www.fastken.com/ \n" \
"for more detail.\n\n", g_fdfs_version.major, g_fdfs_version.minor,
g_fdfs_version.patch);
if (argc < 3)
{
printf("Usage: %s " \
"[FILE | BUFF | CALLBACK]\n", argv[0]);
return 1;
}
log_init();
//g_log_context.log_level = LOG_DEBUG;
conf_filename = argv[1];
if ((result=fdfs_client_init(conf_filename)) != 0)
{
return result;
}
pTrackerServer = tracker_get_connection();
if (pTrackerServer == NULL)
{
fdfs_client_destroy();
return errno != 0 ? errno : ECONNREFUSED;
}
local_filename = argv[2];
if (argc == 3)
{
upload_type = FDFS_UPLOAD_BY_FILE;
}
else
{
if (strcmp(argv[3], "BUFF") == 0)
{
upload_type = FDFS_UPLOAD_BY_BUFF;
}
else if (strcmp(argv[3], "CALLBACK") == 0)
{
upload_type = FDFS_UPLOAD_BY_CALLBACK;
}
else
{
upload_type = FDFS_UPLOAD_BY_FILE;
}
}
store_path_index = 0;
*group_name = '\0';
if ((result=tracker_query_storage_store(pTrackerServer, \
&storageServer, group_name, &store_path_index)) != 0)
{
fdfs_client_destroy();
printf("tracker_query_storage fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
return result;
}
printf("group_name=%s, ip_addr=%s, port=%d\n", \
group_name, storageServer.ip_addr, \
storageServer.port);
if ((pStorageServer=tracker_make_connection(&storageServer, \
&result)) == NULL)
{
fdfs_client_destroy();
return result;
}
memset(&meta_list, 0, sizeof(meta_list));
meta_count = 0;
strcpy(meta_list[meta_count].name, "ext_name");
strcpy(meta_list[meta_count].value, "jpg");
meta_count++;
strcpy(meta_list[meta_count].name, "width");
strcpy(meta_list[meta_count].value, "160");
meta_count++;
strcpy(meta_list[meta_count].name, "height");
strcpy(meta_list[meta_count].value, "80");
meta_count++;
strcpy(meta_list[meta_count].name, "file_size");
strcpy(meta_list[meta_count].value, "115120");
meta_count++;
file_ext_name = fdfs_get_file_ext_name(local_filename);
if (upload_type == FDFS_UPLOAD_BY_FILE)
{
if (stat(local_filename, &stat_buf) == 0 && \
S_ISREG(stat_buf.st_mode))
{
file_size = stat_buf.st_size;
result = storage_upload_appender_by_filename1( \
pTrackerServer, pStorageServer, \
store_path_index, local_filename, \
file_ext_name, meta_list, meta_count, \
group_name, file_id);
}
else
{
result = errno != 0 ? errno : ENOENT;
}
printf("storage_upload_appender_by_filename1\n");
}
else if (upload_type == FDFS_UPLOAD_BY_BUFF)
{
char *file_content;
if ((result=getFileContent(local_filename, \
&file_content, &file_size)) == 0)
{
result = storage_upload_appender_by_filebuff1( \
pTrackerServer, pStorageServer, \
store_path_index, file_content, \
file_size, file_ext_name, \
meta_list, meta_count, \
group_name, file_id);
free(file_content);
}
printf("storage_upload_appender_by_filebuff1\n");
}
else
{
if (stat(local_filename, &stat_buf) == 0 && \
S_ISREG(stat_buf.st_mode))
{
file_size = stat_buf.st_size;
result = storage_upload_appender_by_callback1( \
pTrackerServer, pStorageServer, \
store_path_index, uploadFileCallback, \
local_filename, file_size, \
file_ext_name, meta_list, meta_count, \
group_name, file_id);
}
else
{
result = errno != 0 ? errno : ENOENT;
}
printf("storage_upload_appender_by_callback1\n");
}
if (result != 0)
{
printf("upload file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
tracker_close_connection_ex(pStorageServer, true);
fdfs_client_destroy();
return result;
}
url_len = sprintf(file_url, "http://%s/%s",
pTrackerServer->ip_addr, file_id);
if (g_anti_steal_token)
{
ts = time(NULL);
fdfs_http_gen_token(&g_anti_steal_secret_key, file_id, \
ts, token);
sprintf(file_url + url_len, "?token=%s&ts=%d", token, (int)ts);
}
printf("fild_id=%s\n", file_id);
fdfs_get_file_info1(file_id, &file_info);
printf("source ip address: %s\n", file_info.source_ip_addr);
printf("file timestamp=%s\n", formatDatetime(
file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \
szDatetime, sizeof(szDatetime)));
printf("file size=%"PRId64"\n", file_info.file_size);
printf("file crc32=%u\n", file_info.crc32);
printf("file url: %s\n", file_url);
strcpy(appender_file_id, file_id);
if (storage_truncate_file1(pTrackerServer, pStorageServer, \
appender_file_id, 0) != 0)
{
printf("truncate file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
tracker_close_connection_ex(pStorageServer, true);
fdfs_client_destroy();
return result;
}
fdfs_get_file_info1(file_id, &file_info);
printf("source ip address: %s\n", file_info.source_ip_addr);
printf("file timestamp=%s\n", formatDatetime(
file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \
szDatetime, sizeof(szDatetime)));
printf("file size=%"PRId64"\n", file_info.file_size);
printf("file crc32=%u\n", file_info.crc32);
printf("file url: %s\n", file_url);
if (file_info.file_size != 0)
{
fprintf(stderr, "file size: %"PRId64 \
" != 0!!!", file_info.file_size);
}
//sleep(70);
if (upload_type == FDFS_UPLOAD_BY_FILE)
{
result = storage_append_by_filename1(pTrackerServer, \
pStorageServer, local_filename,
appender_file_id);
printf("storage_append_by_filename\n");
}
else if (upload_type == FDFS_UPLOAD_BY_BUFF)
{
char *file_content;
if ((result=getFileContent(local_filename, \
&file_content, &file_size)) == 0)
{
result = storage_append_by_filebuff1(pTrackerServer, \
pStorageServer, file_content, \
file_size, appender_file_id);
free(file_content);
}
printf("storage_append_by_filebuff1\n");
}
else
{
if (stat(local_filename, &stat_buf) == 0 && \
S_ISREG(stat_buf.st_mode))
{
file_size = stat_buf.st_size;
result = storage_append_by_callback1(pTrackerServer, \
pStorageServer, uploadFileCallback, \
local_filename, file_size, \
appender_file_id);
}
else
{
result = errno != 0 ? errno : ENOENT;
}
printf("storage_append_by_callback1\n");
}
if (result != 0)
{
printf("append file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
tracker_close_connection_ex(pStorageServer, true);
fdfs_client_destroy();
return result;
}
printf("append file successfully.\n");
fdfs_get_file_info1(appender_file_id, &file_info);
printf("source ip address: %s\n", file_info.source_ip_addr);
printf("file timestamp=%s\n", formatDatetime(
file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \
szDatetime, sizeof(szDatetime)));
printf("file size=%"PRId64"\n", file_info.file_size);
if (file_info.file_size != file_size)
{
fprintf(stderr, "file size: %"PRId64 \
" != %"PRId64"!!!", file_info.file_size, \
file_size);
}
file_offset = file_size;
if (upload_type == FDFS_UPLOAD_BY_FILE)
{
result = storage_modify_by_filename1(pTrackerServer, \
pStorageServer, local_filename,
file_offset, appender_file_id);
printf("storage_modify_by_filename\n");
}
else if (upload_type == FDFS_UPLOAD_BY_BUFF)
{
char *file_content;
if ((result=getFileContent(local_filename, \
&file_content, &file_size)) == 0)
{
result = storage_modify_by_filebuff1( \
pTrackerServer, pStorageServer, \
file_content, file_offset, file_size, \
appender_file_id);
free(file_content);
}
printf("storage_modify_by_filebuff1\n");
}
else
{
if (stat(local_filename, &stat_buf) == 0 && \
S_ISREG(stat_buf.st_mode))
{
file_size = stat_buf.st_size;
result = storage_modify_by_callback1( \
pTrackerServer, pStorageServer, \
uploadFileCallback, \
local_filename, file_offset, \
file_size, appender_file_id);
}
else
{
result = errno != 0 ? errno : ENOENT;
}
printf("storage_modify_by_callback1\n");
}
if (result != 0)
{
printf("modify file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
tracker_close_connection_ex(pStorageServer, true);
fdfs_client_destroy();
return result;
}
printf("modify file successfully.\n");
fdfs_get_file_info1(appender_file_id, &file_info);
printf("source ip address: %s\n", file_info.source_ip_addr);
printf("file timestamp=%s\n", formatDatetime(
file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \
szDatetime, sizeof(szDatetime)));
printf("file size=%"PRId64"\n", file_info.file_size);
if (file_info.file_size != 2 * file_size)
{
fprintf(stderr, "file size: %"PRId64 \
" != %"PRId64"!!!", file_info.file_size, \
2 * file_size);
}
tracker_close_connection_ex(pStorageServer, true);
tracker_close_connection_ex(pTrackerServer, true);
fdfs_client_destroy();
return result;
}
================================================
FILE: client/fdfs_bulk_import.c
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include "fdfs_client.h"
#include "fastcommon/logger.h"
#include "fastcommon/shared_func.h"
#include "fastcommon/sched_thread.h"
#include "../storage/storage_bulk_import.h"
#define DEFAULT_THREADS 4
#define MAX_THREADS 32
#define OUTPUT_BUFFER_SIZE 1024
typedef struct {
char config_file[MAX_PATH_SIZE];
char source_path[MAX_PATH_SIZE];
char group_name[FDFS_GROUP_NAME_MAX_LEN + 1];
char output_file[MAX_PATH_SIZE];
int store_path_index;
int import_mode;
int thread_count;
bool recursive;
bool dry_run;
bool calculate_crc32;
bool verbose;
} BulkImportOptions;
static void usage(const char *program_name)
{
printf("FastDFS Bulk Import Tool v1.0\n");
printf("Usage: %s [OPTIONS] \n\n", program_name);
printf("Options:\n");
printf(" -c, --config FastDFS client config file (required)\n");
printf(" -g, --group Target storage group name (required)\n");
printf(" -p, --path-index Storage path index (default: 0)\n");
printf(" -m, --mode Import mode (default: copy)\n");
printf(" -t, --threads Number of worker threads (default: %d, max: %d)\n",
DEFAULT_THREADS, MAX_THREADS);
printf(" -r, --recursive Recursively import directories\n");
printf(" -o, --output Output mapping file (source -> file_id)\n");
printf(" -n, --dry-run Validate only, don't import\n");
printf(" -C, --no-crc32 Skip CRC32 calculation (faster but less safe)\n");
printf(" -v, --verbose Verbose output\n");
printf(" -h, --help Show this help message\n\n");
printf("Examples:\n");
printf(" # Import single file\n");
printf(" %s -c /etc/fdfs/client.conf -g group1 /data/file.jpg\n\n", program_name);
printf(" # Import directory recursively with 8 threads\n");
printf(" %s -c /etc/fdfs/client.conf -g group1 -r -t 8 /data/images/\n\n", program_name);
printf(" # Move files instead of copy\n");
printf(" %s -c /etc/fdfs/client.conf -g group1 -m move /data/old/\n\n", program_name);
printf(" # Dry-run to validate before actual import\n");
printf(" %s -c /etc/fdfs/client.conf -g group1 -n /data/test/\n\n", program_name);
}
static int parse_import_mode(const char *mode_str)
{
if (strcmp(mode_str, "copy") == 0) {
return BULK_IMPORT_MODE_COPY;
} else if (strcmp(mode_str, "move") == 0) {
return BULK_IMPORT_MODE_MOVE;
} else {
return -1;
}
}
static int parse_options(int argc, char *argv[], BulkImportOptions *options)
{
int c;
int option_index = 0;
static struct option long_options[] = {
{"config", required_argument, 0, 'c'},
{"group", required_argument, 0, 'g'},
{"path-index", required_argument, 0, 'p'},
{"mode", required_argument, 0, 'm'},
{"threads", required_argument, 0, 't'},
{"recursive", no_argument, 0, 'r'},
{"output", required_argument, 0, 'o'},
{"dry-run", no_argument, 0, 'n'},
{"no-crc32", no_argument, 0, 'C'},
{"verbose", no_argument, 0, 'v'},
{"help", no_argument, 0, 'h'},
{0, 0, 0, 0}
};
memset(options, 0, sizeof(BulkImportOptions));
options->store_path_index = 0;
options->import_mode = BULK_IMPORT_MODE_COPY;
options->thread_count = DEFAULT_THREADS;
options->calculate_crc32 = true;
while ((c = getopt_long(argc, argv, "c:g:p:m:t:ro:nCvh", long_options, &option_index)) != -1) {
switch (c) {
case 'c':
snprintf(options->config_file, sizeof(options->config_file), "%s", optarg);
break;
case 'g':
snprintf(options->group_name, sizeof(options->group_name), "%s", optarg);
break;
case 'p':
options->store_path_index = atoi(optarg);
break;
case 'm':
options->import_mode = parse_import_mode(optarg);
if (options->import_mode < 0) {
fprintf(stderr, "Invalid import mode: %s (use 'copy' or 'move')\n", optarg);
return EINVAL;
}
break;
case 't':
options->thread_count = atoi(optarg);
if (options->thread_count < 1 || options->thread_count > MAX_THREADS) {
fprintf(stderr, "Thread count must be between 1 and %d\n", MAX_THREADS);
return EINVAL;
}
break;
case 'r':
options->recursive = true;
break;
case 'o':
snprintf(options->output_file, sizeof(options->output_file), "%s", optarg);
break;
case 'n':
options->dry_run = true;
break;
case 'C':
options->calculate_crc32 = false;
break;
case 'v':
options->verbose = true;
break;
case 'h':
usage(argv[0]);
exit(0);
default:
return EINVAL;
}
}
if (optind >= argc) {
fprintf(stderr, "Error: Source path is required\n\n");
usage(argv[0]);
return EINVAL;
}
snprintf(options->source_path, sizeof(options->source_path), "%s", argv[optind]);
if (options->config_file[0] == '\0') {
fprintf(stderr, "Error: Config file is required (-c option)\n\n");
usage(argv[0]);
return EINVAL;
}
if (options->group_name[0] == '\0') {
fprintf(stderr, "Error: Group name is required (-g option)\n\n");
usage(argv[0]);
return EINVAL;
}
return 0;
}
static int write_output_mapping(FILE *fp, const BulkImportFileInfo *file_info)
{
if (fp == NULL || file_info == NULL) {
return EINVAL;
}
fprintf(fp, "%s\t%s\t%"PRId64"\t%u\t%s\n",
file_info->source_path,
file_info->file_id,
file_info->file_size,
file_info->crc32,
file_info->status == BULK_IMPORT_STATUS_SUCCESS ? "SUCCESS" :
file_info->status == BULK_IMPORT_STATUS_FAILED ? "FAILED" : "SKIPPED");
fflush(fp);
return 0;
}
static int import_single_file(BulkImportContext *context,
const BulkImportOptions *options, const char *file_path, FILE *output_fp)
{
BulkImportFileInfo file_info;
int result;
if (options->verbose) {
printf("Processing: %s\n", file_path);
}
result = storage_calculate_file_metadata(file_path, &file_info, options->calculate_crc32);
if (result != 0) {
fprintf(stderr, "Error calculating metadata for %s: %s\n",
file_path, file_info.error_message);
__sync_add_and_fetch(&context->failed_files, 1);
return result;
}
result = storage_generate_file_id(&file_info, options->group_name, options->store_path_index);
if (result != 0) {
fprintf(stderr, "Error generating file ID for %s: %s\n",
file_path, file_info.error_message);
__sync_add_and_fetch(&context->failed_files, 1);
return result;
}
result = storage_register_bulk_file(context, &file_info);
if (result != 0) {
fprintf(stderr, "Error importing %s: %s\n",
file_path, file_info.error_message);
__sync_add_and_fetch(&context->failed_files, 1);
} else {
if (options->verbose) {
printf(" -> %s (%"PRId64" bytes)\n", file_info.file_id, file_info.file_size);
}
}
if (output_fp != NULL) {
write_output_mapping(output_fp, &file_info);
}
__sync_add_and_fetch(&context->processed_files, 1);
return result;
}
static int import_directory_recursive(BulkImportContext *context,
const BulkImportOptions *options, const char *dir_path, FILE *output_fp)
{
DIR *dir;
struct dirent *entry;
struct stat st;
char full_path[MAX_PATH_SIZE];
int result = 0;
int file_result;
dir = opendir(dir_path);
if (dir == NULL) {
fprintf(stderr, "Error opening directory %s: %s\n", dir_path, STRERROR(errno));
return errno != 0 ? errno : EIO;
}
while ((entry = readdir(dir)) != NULL) {
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) {
continue;
}
snprintf(full_path, sizeof(full_path), "%s/%s", dir_path, entry->d_name);
if (stat(full_path, &st) != 0) {
fprintf(stderr, "Error stat %s: %s\n", full_path, STRERROR(errno));
continue;
}
if (S_ISREG(st.st_mode)) {
__sync_add_and_fetch(&context->total_files, 1);
file_result = import_single_file(context, options, full_path, output_fp);
if (file_result != 0 && result == 0) {
result = file_result;
}
} else if (S_ISDIR(st.st_mode) && options->recursive) {
file_result = import_directory_recursive(context, options, full_path, output_fp);
if (file_result != 0 && result == 0) {
result = file_result;
}
}
}
closedir(dir);
return result;
}
static void print_summary(const BulkImportContext *context, const BulkImportOptions *options)
{
time_t duration = context->end_time - context->start_time;
double speed_mbps = 0.0;
if (duration > 0) {
speed_mbps = (double)context->total_bytes / (1024.0 * 1024.0) / duration;
}
printf("\n");
printf("=== Import Summary ===\n");
printf("Mode: %s\n", options->dry_run ? "DRY-RUN" :
(options->import_mode == BULK_IMPORT_MODE_COPY ? "COPY" : "MOVE"));
printf("Total files: %"PRId64"\n", context->total_files);
printf("Processed: %"PRId64"\n", context->processed_files);
printf("Success: %"PRId64"\n", context->success_files);
printf("Failed: %"PRId64"\n", context->failed_files);
printf("Skipped: %"PRId64"\n", context->skipped_files);
printf("Total bytes: %"PRId64" (%.2f GB)\n",
context->total_bytes, (double)context->total_bytes / (1024.0 * 1024.0 * 1024.0));
printf("Duration: %"PRId64" seconds\n", (int64_t)duration);
printf("Speed: %.2f MB/s\n", speed_mbps);
printf("======================\n");
}
int main(int argc, char *argv[])
{
BulkImportOptions options;
BulkImportContext context;
struct stat st;
FILE *output_fp = NULL;
int result;
if (argc < 2) {
usage(argv[0]);
return 1;
}
result = parse_options(argc, argv, &options);
if (result != 0) {
return result;
}
log_init();
if (options.verbose) {
g_log_context.log_level = LOG_DEBUG;
} else {
g_log_context.log_level = LOG_INFO;
}
printf("FastDFS Bulk Import Tool\n");
printf("Config: %s\n", options.config_file);
printf("Group: %s\n", options.group_name);
printf("Source: %s\n", options.source_path);
printf("Mode: %s\n", options.dry_run ? "DRY-RUN" :
(options.import_mode == BULK_IMPORT_MODE_COPY ? "COPY" : "MOVE"));
printf("Threads: %d\n", options.thread_count);
printf("CRC32: %s\n", options.calculate_crc32 ? "enabled" : "disabled");
printf("\n");
result = storage_bulk_import_init();
if (result != 0) {
fprintf(stderr, "Failed to initialize bulk import module\n");
return result;
}
memset(&context, 0, sizeof(context));
snprintf(context.group_name, sizeof(context.group_name), "%s", options.group_name);
context.store_path_index = options.store_path_index;
context.import_mode = options.import_mode;
context.calculate_crc32 = options.calculate_crc32;
context.validate_only = options.dry_run;
context.start_time = time(NULL);
if (options.output_file[0] != '\0') {
output_fp = fopen(options.output_file, "w");
if (output_fp == NULL) {
fprintf(stderr, "Error opening output file %s: %s\n",
options.output_file, STRERROR(errno));
storage_bulk_import_destroy();
return errno != 0 ? errno : EIO;
}
fprintf(output_fp, "# Source\tFileID\tSize\tCRC32\tStatus\n");
}
if (stat(options.source_path, &st) != 0) {
fprintf(stderr, "Error: Source path not found: %s\n", options.source_path);
if (output_fp != NULL) {
fclose(output_fp);
}
storage_bulk_import_destroy();
return errno != 0 ? errno : ENOENT;
}
if (S_ISREG(st.st_mode)) {
context.total_files = 1;
result = import_single_file(&context, &options, options.source_path, output_fp);
} else if (S_ISDIR(st.st_mode)) {
result = import_directory_recursive(&context, &options, options.source_path, output_fp);
} else {
fprintf(stderr, "Error: Source path is not a file or directory\n");
result = EINVAL;
}
context.end_time = time(NULL);
if (output_fp != NULL) {
fclose(output_fp);
printf("Output mapping written to: %s\n", options.output_file);
}
print_summary(&context, &options);
storage_bulk_import_destroy();
return result == 0 ? 0 : 1;
}
================================================
FILE: client/fdfs_client.h
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#ifndef FDFS_CLIENT_H
#define FDFS_CLIENT_H
#include "fastcommon/shared_func.h"
#include "tracker_types.h"
#include "tracker_proto.h"
#include "tracker_client.h"
#include "storage_client.h"
#include "storage_client1.h"
#include "client_func.h"
#include "client_global.h"
#endif
================================================
FILE: client/fdfs_crc32.c
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include
#include
#include
#include
#include
#include
#include
#include
#include "fastcommon/hash.h"
int main(int argc, char *argv[])
{
int64_t file_size;
int64_t remain_bytes;
char *filename;
int fd;
int read_bytes;
int result;
int64_t crc32;
char buff[512 * 1024];
if (argc < 2)
{
printf("Usage: %s \n", argv[0]);
return 1;
}
filename = argv[1];
fd = open(filename, O_RDONLY);
if (fd < 0)
{
printf("file: "__FILE__", line: %d, " \
"open file %s fail, " \
"errno: %d, error info: %s\n", \
__LINE__, filename, errno, STRERROR(errno));
return errno != 0 ? errno : EACCES;
}
if ((file_size=lseek(fd, 0, SEEK_END)) < 0)
{
printf("file: "__FILE__", line: %d, " \
"call lseek fail, " \
"errno: %d, error info: %s\n", \
__LINE__, errno, STRERROR(errno));
close(fd);
return errno;
}
if (lseek(fd, 0, SEEK_SET) < 0)
{
printf("file: "__FILE__", line: %d, " \
"call lseek fail, " \
"errno: %d, error info: %s\n", \
__LINE__, errno, STRERROR(errno));
close(fd);
return errno;
}
crc32 = CRC32_XINIT;
result = 0;
remain_bytes = file_size;
while (remain_bytes > 0)
{
if (remain_bytes > sizeof(buff))
{
read_bytes = sizeof(buff);
}
else
{
read_bytes = remain_bytes;
}
if (read(fd, buff, read_bytes) != read_bytes)
{
printf("file: "__FILE__", line: %d, " \
"call lseek fail, " \
"errno: %d, error info: %s\n", \
__LINE__, errno, STRERROR(errno));
result = errno != 0 ? errno : EIO;
break;
}
crc32 = CRC32_ex(buff, read_bytes, crc32);
remain_bytes -= read_bytes;
}
close(fd);
if (result == 0)
{
crc32 = CRC32_FINAL(crc32);
printf("%x\n", (int)crc32);
}
return result;
}
================================================
FILE: client/fdfs_delete_file.c
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include
#include
#include
#include
#include
#include
#include
#include "fdfs_client.h"
#include "fastcommon/logger.h"
int main(int argc, char *argv[])
{
char *conf_filename;
ConnectionInfo *pTrackerServer;
int result;
char file_id[128];
if (argc < 3)
{
printf("Usage: %s \n", argv[0]);
return 1;
}
log_init();
g_log_context.log_level = LOG_ERR;
ignore_signal_pipe();
conf_filename = argv[1];
if ((result=fdfs_client_init(conf_filename)) != 0)
{
return result;
}
pTrackerServer = tracker_get_connection();
if (pTrackerServer == NULL)
{
fdfs_client_destroy();
return errno != 0 ? errno : ECONNREFUSED;
}
fc_safe_strcpy(file_id, argv[2]);
if ((result=storage_delete_file1(pTrackerServer, NULL, file_id)) != 0)
{
printf("delete file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
}
tracker_close_connection_ex(pTrackerServer, true);
fdfs_client_destroy();
return result;
}
================================================
FILE: client/fdfs_download_file.c
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include
#include
#include
#include
#include
#include
#include
#include "fdfs_client.h"
#include "fastcommon/logger.h"
int main(int argc, char *argv[])
{
char *conf_filename;
char *local_filename;
ConnectionInfo *pTrackerServer;
int result;
char file_id[128];
int64_t file_size;
int64_t file_offset;
int64_t download_bytes;
if (argc < 3)
{
printf("Usage: %s " \
"[local_filename] [ " \
"]\n", argv[0]);
return 1;
}
log_init();
g_log_context.log_level = LOG_ERR;
ignore_signal_pipe();
conf_filename = argv[1];
if ((result=fdfs_client_init(conf_filename)) != 0)
{
return result;
}
pTrackerServer = tracker_get_connection();
if (pTrackerServer == NULL)
{
fdfs_client_destroy();
return errno != 0 ? errno : ECONNREFUSED;
}
fc_safe_strcpy(file_id, argv[2]);
file_offset = 0;
download_bytes = 0;
if (argc >= 4)
{
local_filename = argv[3];
if (argc >= 6)
{
file_offset = strtoll(argv[4], NULL, 10);
download_bytes = strtoll(argv[5], NULL, 10);
}
}
else
{
local_filename = strrchr(file_id, '/');
if (local_filename != NULL)
{
local_filename++; //skip /
}
else
{
local_filename = file_id;
}
}
result = storage_do_download_file1_ex(pTrackerServer, \
NULL, FDFS_DOWNLOAD_TO_FILE, file_id, \
file_offset, download_bytes, \
&local_filename, NULL, &file_size);
if (result != 0)
{
printf("download file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
}
tracker_close_connection_ex(pTrackerServer, true);
fdfs_client_destroy();
return 0;
}
================================================
FILE: client/fdfs_file_info.c
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include
#include
#include
#include
#include
#include
#include
#include "fdfs_client.h"
#include "fastcommon/logger.h"
static void usage(const char *program)
{
fprintf(stderr, "Usage: %s [options] \n"
" options: \n"
" -s: keep silence, when this file not exist, "
"do not log error on storage server\n"
" -n: do NOT calculate CRC32 for appender file "
"or slave file\n\n", program);
}
int main(int argc, char *argv[])
{
char *conf_filename;
const char *file_type_str;
char file_id[128];
int result;
int ch;
char flags;
FDFSFileInfo file_info;
if (argc < 3)
{
usage(argv[0]);
return 1;
}
flags = 0;
while ((ch=getopt(argc, argv, "ns")) != -1) {
switch (ch) {
case 'n':
flags |= FDFS_QUERY_FINFO_FLAGS_NOT_CALC_CRC32;
break;
case 's':
flags |= FDFS_QUERY_FINFO_FLAGS_KEEP_SILENCE;
break;
default:
usage(argv[0]);
return 1;
}
}
if (optind + 2 > argc) {
usage(argv[0]);
return 1;
}
log_init();
g_log_context.log_level = LOG_ERR;
ignore_signal_pipe();
conf_filename = argv[optind];
if ((result=fdfs_client_init(conf_filename)) != 0)
{
return result;
}
fc_safe_strcpy(file_id, argv[optind+1]);
memset(&file_info, 0, sizeof(file_info));
result = fdfs_get_file_info_ex1(file_id, true, &file_info, flags);
if (result != 0)
{
fprintf(stderr, "query file info fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
}
else
{
char szDatetime[32];
switch (file_info.file_type)
{
case FDFS_FILE_TYPE_NORMAL:
file_type_str = "normal";
break;
case FDFS_FILE_TYPE_SLAVE:
file_type_str = "slave";
break;
case FDFS_FILE_TYPE_APPENDER:
file_type_str = "appender";
break;
default:
file_type_str = "unknown";
break;
}
printf("GET FROM SERVER: %s\n\n",
file_info.get_from_server ? "true" : "false");
printf("file type: %s\n", file_type_str);
printf("source storage id: %d\n", file_info.source_id);
printf("source ip address: %s\n", file_info.source_ip_addr);
printf("file create timestamp: %s\n", formatDatetime(
file_info.create_timestamp, "%Y-%m-%d %H:%M:%S",
szDatetime, sizeof(szDatetime)));
printf("file size: %"PRId64"\n", file_info.file_size);
if ((flags & FDFS_QUERY_FINFO_FLAGS_NOT_CALC_CRC32) == 0 ||
file_info.crc32 != 0)
{
printf("file crc32: %d (0x%08x)\n",
file_info.crc32, file_info.crc32);
}
printf("\n");
}
tracker_close_all_connections();
fdfs_client_destroy();
return 0;
}
================================================
FILE: client/fdfs_link_library.sh.in
================================================
tmp_src_filename=_fdfs_check_bits_.c
cat < $tmp_src_filename
#include
#include
#include
int main()
{
printf("%d\n", (int)sizeof(long));
return 0;
}
EOF
gcc -D_FILE_OFFSET_BITS=64 -o a.out $tmp_src_filename
OS_BITS=`./a.out`
rm $tmp_src_filename a.out
TARGET_LIB="$(TARGET_PREFIX)/lib"
if [ "`id -u`" = "0" ]; then
ln -fs $TARGET_LIB/libfastcommon.so.1 /usr/lib/libfastcommon.so
ln -fs $TARGET_LIB/libfdfsclient.so.1 /usr/lib/libfdfsclient.so
if [ "$OS_BITS" = "8" ]; then
ln -fs $TARGET_LIB/libfastcommon.so.1 /usr/lib64/libfastcommon.so
ln -fs $TARGET_LIB/libfdfsclient.so.1 /usr/lib64/libfdfsclient.so
fi
fi
================================================
FILE: client/fdfs_monitor.c
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include
#include
#include
#include
#include
#include
#include
#include
#include "fastcommon/sockopt.h"
#include "fastcommon/logger.h"
#include "client_global.h"
#include "fdfs_global.h"
#include "fdfs_client.h"
#define MB_TO_HUMAN_STR(mb, buff) bytes_to_human_str( \
(int64_t)mb * FC_BYTES_ONE_MB, buff)
static ConnectionInfo *pTrackerServer;
static int list_all_groups(const char *group_name);
static void usage(char *argv[])
{
printf("Usage: %s [-h ] "
"[list|delete|set_trunk_server [storage_id]]\n"
"\tthe tracker server format: host[:port], "
"the tracker default port is %d\n\n",
argv[0], FDFS_TRACKER_SERVER_DEF_PORT);
}
int main(int argc, char *argv[])
{
char formatted_ip[FORMATTED_IP_SIZE];
char *conf_filename;
char *op_type;
char *group_name;
char *tracker_server;
int arg_index;
int result;
if (argc < 2)
{
usage(argv);
return 1;
}
tracker_server = NULL;
conf_filename = argv[1];
arg_index = 2;
if (arg_index >= argc)
{
op_type = "list";
}
else
{
int len;
len = strlen(argv[arg_index]);
if (len >= 2 && strncmp(argv[arg_index], "-h", 2) == 0)
{
if (len == 2)
{
arg_index++;
if (arg_index >= argc)
{
usage(argv);
return 1;
}
tracker_server = argv[arg_index++];
}
else
{
tracker_server = argv[arg_index] + 2;
arg_index++;
}
if (arg_index < argc)
{
op_type = argv[arg_index++];
}
else
{
op_type = "list";
}
}
else
{
op_type = argv[arg_index++];
}
}
log_init();
//g_log_context.log_level = LOG_DEBUG;
ignore_signal_pipe();
if ((result=fdfs_client_init(conf_filename)) != 0)
{
return result;
}
load_log_level_ex(conf_filename);
if (tracker_server == NULL)
{
if (g_tracker_group.server_count > 1)
{
srand(time(NULL));
rand(); //discard the first
g_tracker_group.server_index = (int)( \
(g_tracker_group.server_count * (double)rand()) \
/ (double)RAND_MAX);
}
}
else
{
int i;
ConnectionInfo conn;
if ((result=conn_pool_parse_server_info(tracker_server, &conn,
FDFS_TRACKER_SERVER_DEF_PORT)) != 0)
{
printf("resolve ip address of tracker server: %s "
"fail!, error info: %s\n", tracker_server, hstrerror(h_errno));
return result;
}
for (i=0; iip_addr, formatted_ip);
printf("\ntracker server is %s:%u\n\n", formatted_ip,
pTrackerServer->port);
if (arg_index < argc)
{
group_name = argv[arg_index++];
}
else
{
group_name = NULL;
}
if (strcmp(op_type, "list") == 0)
{
if (group_name == NULL)
{
result = list_all_groups(NULL);
}
else
{
result = list_all_groups(group_name);
}
}
else if (strcmp(op_type, "delete") == 0)
{
if (arg_index >= argc)
{
if ((result=tracker_delete_group(&g_tracker_group, \
group_name)) == 0)
{
printf("delete group: %s success\n", \
group_name);
}
else
{
printf("delete group: %s fail, " \
"error no: %d, error info: %s\n", \
group_name, result, STRERROR(result));
}
}
else
{
char *storage_id;
storage_id = argv[arg_index++];
if ((result=tracker_delete_storage(&g_tracker_group, \
group_name, storage_id)) == 0)
{
printf("delete storage server %s::%s success\n", \
group_name, storage_id);
}
else
{
printf("delete storage server %s::%s fail, " \
"error no: %d, error info: %s\n", \
group_name, storage_id, \
result, STRERROR(result));
}
}
}
else if (strcmp(op_type, "set_trunk_server") == 0)
{
char *storage_id;
char new_trunk_server_id[FDFS_STORAGE_ID_MAX_SIZE];
if (group_name == NULL)
{
usage(argv);
return 1;
}
if (arg_index >= argc)
{
storage_id = "";
}
else
{
storage_id = argv[arg_index++];
}
if ((result=tracker_set_trunk_server(&g_tracker_group, \
group_name, storage_id, new_trunk_server_id)) == 0)
{
printf("set trunk server %s::%s success, " \
"new trunk server: %s\n", group_name, \
storage_id, new_trunk_server_id);
}
else
{
printf("set trunk server %s::%s fail, " \
"error no: %d, error info: %s\n", \
group_name, storage_id, \
result, STRERROR(result));
}
}
else
{
printf("Invalid command %s\n\n", op_type);
usage(argv);
}
tracker_close_connection_ex(pTrackerServer, true);
fdfs_client_destroy();
return 0;
}
static const char *get_storage_rw_caption(const FDFSReadWriteMode rw_mode)
{
switch (rw_mode)
{
case fdfs_rw_none:
return "none (disabled)";
case fdfs_rw_readonly:
return "readonly";
case fdfs_rw_writeonly:
return "writeonly";
case fdfs_rw_both:
return "both (normal)";
default:
return "unknown";
}
}
static int list_storages(FDFSGroupStat *pGroupStat)
{
int result;
int storage_count;
int k;
int max_last_source_update;
int64_t avail_space;
FDFSStorageInfo storage_infos[FDFS_MAX_SERVERS_EACH_GROUP];
FDFSStorageInfo *p;
FDFSStorageInfo *pStorage;
FDFSStorageInfo *pStorageEnd;
FDFSStorageStat *pStorageStat;
char szJoinTime[32];
char szUpTime[32];
char szLastHeartBeatTime[32];
char szSrcUpdTime[32];
char szSyncUpdTime[32];
char szSyncedTimestamp[32];
char szSyncedDelaySeconds[128];
char szHostname[128];
char szHostnamePrompt[128+8];
char szDiskTotalSpace[32];
char szDiskFreeSpace[32];
char szDiskReservedSpace[32];
char szDiskAvailSpace[32];
char szTrunkSpace[32];
result = tracker_list_servers(pTrackerServer, pGroupStat->group_name,
NULL, storage_infos, FDFS_MAX_SERVERS_EACH_GROUP, &storage_count);
if (result != 0)
{
return result;
}
avail_space = pGroupStat->free_mb - pGroupStat->reserved_mb;
if (avail_space < 0)
{
avail_space = 0;
}
printf( "group name = %s\n"
"disk total space = %7s\n"
"disk free space = %7s\n"
"disk reserved space = %7s\n"
"disk available space = %7s\n",
pGroupStat->group_name,
MB_TO_HUMAN_STR(pGroupStat->total_mb, szDiskTotalSpace),
MB_TO_HUMAN_STR(pGroupStat->free_mb, szDiskFreeSpace),
MB_TO_HUMAN_STR(pGroupStat->reserved_mb, szDiskReservedSpace),
MB_TO_HUMAN_STR(avail_space, szDiskAvailSpace));
if (pGroupStat->current_trunk_file_id >= 0) //use trunk file
{
printf("trunk free space = %7s\n", MB_TO_HUMAN_STR(
pGroupStat->trunk_free_mb, szTrunkSpace));
}
printf( "storage server count = %d\n"
"readable server count = %d\n"
"writable server count = %d\n"
"storage server port = %d\n"
"store path count = %d\n"
"subdir count per path = %d\n"
"current write server index = %d\n",
pGroupStat->storage_count,
pGroupStat->readable_server_count,
pGroupStat->writable_server_count,
pGroupStat->storage_port,
pGroupStat->store_path_count,
pGroupStat->subdir_count_per_path,
pGroupStat->current_write_server);
pStorageEnd = storage_infos + storage_count;
if (pGroupStat->current_trunk_file_id >= 0) //use trunk file
{
for (pStorage=storage_infos; pStorageif_trunk_server)
{
break;
}
}
if (pStorage < pStorageEnd) //found trunk server
{
printf( "current trunk server = %s (%s)\n"
"current trunk file id = %d\n\n",
pStorage->id, pStorage->ip_addr,
pGroupStat->current_trunk_file_id);
}
else
{
printf("\n");
}
}
else
{
printf("\n");
}
k = 0;
for (pStorage=storage_infos; pStoragestat.last_source_update
> max_last_source_update)
{
max_last_source_update = \
p->stat.last_source_update;
}
}
pStorageStat = &(pStorage->stat);
if (max_last_source_update == 0)
{
*szSyncedDelaySeconds = '\0';
}
else
{
if (pStorageStat->last_synced_timestamp == 0)
{
strcpy(szSyncedDelaySeconds, "(never synced)");
}
else
{
int delay_seconds;
int remain_seconds;
int day;
int hour;
int minute;
int second;
char szDelayTime[64];
delay_seconds = (int)(max_last_source_update -
pStorageStat->last_synced_timestamp);
if (delay_seconds < 0)
{
delay_seconds = 0;
}
day = delay_seconds / (24 * 3600);
remain_seconds = delay_seconds % (24 * 3600);
hour = remain_seconds / 3600;
remain_seconds %= 3600;
minute = remain_seconds / 60;
second = remain_seconds % 60;
if (day != 0)
{
sprintf(szDelayTime, "%d days " \
"%02dh:%02dm:%02ds", \
day, hour, minute, second);
}
else if (hour != 0)
{
sprintf(szDelayTime, "%02dh:%02dm:%02ds", \
hour, minute, second);
}
else if (minute != 0)
{
sprintf(szDelayTime, "%02dm:%02ds", minute, second);
}
else
{
sprintf(szDelayTime, "%ds", second);
}
sprintf(szSyncedDelaySeconds, "(%s delay)", szDelayTime);
}
}
//getHostnameByIp(pStorage->ip_addr, szHostname, sizeof(szHostname));
*szHostname = '\0';
if (*szHostname != '\0')
{
sprintf(szHostnamePrompt, " (%s)", szHostname);
}
else
{
*szHostnamePrompt = '\0';
}
if (pStorage->up_time != 0)
{
formatDatetime(pStorage->up_time, \
"%Y-%m-%d %H:%M:%S", \
szUpTime, sizeof(szUpTime));
}
else
{
*szUpTime = '\0';
}
avail_space = pStorage->free_mb - pStorage->reserved_mb;
if (avail_space < 0)
{
avail_space = 0;
}
printf( "\tStorage %d:\n"
"\t\tid = %s\n"
"\t\tip_addr = %s%s %s\n"
"\t\tread write mode = %s\n"
"\t\tversion = %s\n"
"\t\tjoin time = %s\n"
"\t\tup time = %s\n"
"\t\tdisk total space = %7s\n"
"\t\tdisk free space = %7s\n"
"\t\tdisk reserved space = %7s\n"
"\t\tdisk available space = %7s\n"
"\t\tupload priority = %d\n"
"\t\tstore_path_count = %d\n"
"\t\tsubdir_count_per_path = %d\n"
"\t\tstorage_port = %d\n"
"\t\tcurrent_write_path = %d\n"
"\t\tsource storage id = %s\n"
"\t\tif_trunk_server = %d\n"
"\t\tconnection.alloc_count = %d\n"
"\t\tconnection.current_count = %d\n"
"\t\tconnection.max_count = %d\n"
"\t\ttotal_upload_count = %"PRId64"\n"
"\t\tsuccess_upload_count = %"PRId64"\n"
"\t\ttotal_append_count = %"PRId64"\n"
"\t\tsuccess_append_count = %"PRId64"\n"
"\t\ttotal_modify_count = %"PRId64"\n"
"\t\tsuccess_modify_count = %"PRId64"\n"
"\t\ttotal_truncate_count = %"PRId64"\n"
"\t\tsuccess_truncate_count = %"PRId64"\n"
"\t\ttotal_set_meta_count = %"PRId64"\n"
"\t\tsuccess_set_meta_count = %"PRId64"\n"
"\t\ttotal_delete_count = %"PRId64"\n"
"\t\tsuccess_delete_count = %"PRId64"\n"
"\t\ttotal_download_count = %"PRId64"\n"
"\t\tsuccess_download_count = %"PRId64"\n"
"\t\ttotal_get_meta_count = %"PRId64"\n"
"\t\tsuccess_get_meta_count = %"PRId64"\n"
"\t\ttotal_create_link_count = %"PRId64"\n"
"\t\tsuccess_create_link_count = %"PRId64"\n"
"\t\ttotal_delete_link_count = %"PRId64"\n"
"\t\tsuccess_delete_link_count = %"PRId64"\n"
"\t\ttotal_upload_bytes = %"PRId64"\n"
"\t\tsuccess_upload_bytes = %"PRId64"\n"
"\t\ttotal_append_bytes = %"PRId64"\n"
"\t\tsuccess_append_bytes = %"PRId64"\n"
"\t\ttotal_modify_bytes = %"PRId64"\n"
"\t\tsuccess_modify_bytes = %"PRId64"\n"
"\t\tstotal_download_bytes = %"PRId64"\n"
"\t\tsuccess_download_bytes = %"PRId64"\n"
"\t\ttotal_sync_in_bytes = %"PRId64"\n"
"\t\tsuccess_sync_in_bytes = %"PRId64"\n"
"\t\ttotal_sync_out_bytes = %"PRId64"\n"
"\t\tsuccess_sync_out_bytes = %"PRId64"\n"
"\t\ttotal_file_open_count = %"PRId64"\n"
"\t\tsuccess_file_open_count = %"PRId64"\n"
"\t\ttotal_file_read_count = %"PRId64"\n"
"\t\tsuccess_file_read_count = %"PRId64"\n"
"\t\ttotal_file_write_count = %"PRId64"\n"
"\t\tsuccess_file_write_count = %"PRId64"\n"
"\t\tlast_heart_beat_time = %s\n"
"\t\tlast_source_update = %s\n"
"\t\tlast_sync_update = %s\n"
"\t\tlast_synced_timestamp = %s %s\n",
++k, pStorage->id, pStorage->ip_addr,
szHostnamePrompt, get_storage_status_caption(
pStorage->status),
get_storage_rw_caption(pStorage->rw_mode),
pStorage->version,
formatDatetime(pStorage->join_time,
"%Y-%m-%d %H:%M:%S",
szJoinTime, sizeof(szJoinTime)), szUpTime,
MB_TO_HUMAN_STR(pStorage->total_mb, szDiskTotalSpace),
MB_TO_HUMAN_STR(pStorage->free_mb, szDiskFreeSpace),
MB_TO_HUMAN_STR(pStorage->reserved_mb, szDiskReservedSpace),
MB_TO_HUMAN_STR(avail_space, szDiskAvailSpace),
pStorage->upload_priority,
pStorage->store_path_count,
pStorage->subdir_count_per_path,
pStorage->storage_port,
pStorage->current_write_path,
pStorage->src_id,
pStorage->if_trunk_server,
pStorageStat->connection.alloc_count,
pStorageStat->connection.current_count,
pStorageStat->connection.max_count,
pStorageStat->total_upload_count,
pStorageStat->success_upload_count,
pStorageStat->total_append_count,
pStorageStat->success_append_count,
pStorageStat->total_modify_count,
pStorageStat->success_modify_count,
pStorageStat->total_truncate_count,
pStorageStat->success_truncate_count,
pStorageStat->total_set_meta_count,
pStorageStat->success_set_meta_count,
pStorageStat->total_delete_count,
pStorageStat->success_delete_count,
pStorageStat->total_download_count,
pStorageStat->success_download_count,
pStorageStat->total_get_meta_count,
pStorageStat->success_get_meta_count,
pStorageStat->total_create_link_count,
pStorageStat->success_create_link_count,
pStorageStat->total_delete_link_count,
pStorageStat->success_delete_link_count,
pStorageStat->total_upload_bytes,
pStorageStat->success_upload_bytes,
pStorageStat->total_append_bytes,
pStorageStat->success_append_bytes,
pStorageStat->total_modify_bytes,
pStorageStat->success_modify_bytes,
pStorageStat->total_download_bytes,
pStorageStat->success_download_bytes,
pStorageStat->total_sync_in_bytes,
pStorageStat->success_sync_in_bytes,
pStorageStat->total_sync_out_bytes,
pStorageStat->success_sync_out_bytes,
pStorageStat->total_file_open_count,
pStorageStat->success_file_open_count,
pStorageStat->total_file_read_count,
pStorageStat->success_file_read_count,
pStorageStat->total_file_write_count,
pStorageStat->success_file_write_count,
formatDatetime(pStorageStat->last_heart_beat_time,
"%Y-%m-%d %H:%M:%S",
szLastHeartBeatTime, sizeof(szLastHeartBeatTime)),
formatDatetime(pStorageStat->last_source_update,
"%Y-%m-%d %H:%M:%S",
szSrcUpdTime, sizeof(szSrcUpdTime)),
formatDatetime(pStorageStat->last_sync_update,
"%Y-%m-%d %H:%M:%S",
szSyncUpdTime, sizeof(szSyncUpdTime)),
formatDatetime(pStorageStat->last_synced_timestamp,
"%Y-%m-%d %H:%M:%S",
szSyncedTimestamp, sizeof(szSyncedTimestamp)),
szSyncedDelaySeconds);
}
return 0;
}
static int list_all_groups(const char *group_name)
{
int result;
int group_count;
FDFSGroupStat group_stats[FDFS_MAX_GROUPS];
FDFSGroupStat *pGroupStat;
FDFSGroupStat *pGroupEnd;
int i;
result = tracker_list_groups(pTrackerServer, group_stats,
FDFS_MAX_GROUPS, &group_count);
if (result != 0)
{
tracker_close_all_connections();
fdfs_client_destroy();
return result;
}
pGroupEnd = group_stats + group_count;
if (group_name == NULL)
{
printf("group count: %d\n", group_count);
i = 0;
for (pGroupStat=group_stats; pGroupStatgroup_name, group_name) == 0)
{
list_storages(pGroupStat);
break;
}
}
}
return 0;
}
================================================
FILE: client/fdfs_regenerate_filename.c
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include
#include
#include
#include
#include
#include
#include
#include "fdfs_client.h"
#include "fastcommon/logger.h"
int main(int argc, char *argv[])
{
char *conf_filename;
ConnectionInfo *pTrackerServer;
int result;
char appender_file_id[128];
char new_file_id[128];
if (argc < 3)
{
fprintf(stderr, "regenerate filename for the appender file.\n"
"NOTE: the regenerated file will be a normal file!\n"
"Usage: %s \n",
argv[0]);
return 1;
}
log_init();
g_log_context.log_level = LOG_ERR;
conf_filename = argv[1];
if ((result=fdfs_client_init(conf_filename)) != 0)
{
return result;
}
pTrackerServer = tracker_get_connection();
if (pTrackerServer == NULL)
{
fdfs_client_destroy();
return errno != 0 ? errno : ECONNREFUSED;
}
fc_safe_strcpy(appender_file_id, argv[2]);
if ((result=storage_regenerate_appender_filename1(pTrackerServer,
NULL, appender_file_id, new_file_id)) != 0)
{
fprintf(stderr, "regenerate file %s fail, "
"error no: %d, error info: %s\n",
appender_file_id, result, STRERROR(result));
return result;
}
printf("%s\n", new_file_id);
tracker_close_connection_ex(pTrackerServer, true);
fdfs_client_destroy();
return result;
}
================================================
FILE: client/fdfs_test.c
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include
#include
#include
#include
#include
#include
#include
#include "fdfs_client.h"
#include "fdfs_global.h"
#include "fastcommon/base64.h"
#include "fastcommon/sockopt.h"
#include "fastcommon/logger.h"
#include "fdfs_http_shared.h"
int writeToFileCallback(void *arg, const int64_t file_size, const char *data, \
const int current_size)
{
if (arg == NULL)
{
return EINVAL;
}
if (fwrite(data, current_size, 1, (FILE *)arg) != 1)
{
return errno != 0 ? errno : EIO;
}
return 0;
}
int uploadFileCallback(void *arg, const int64_t file_size, int sock)
{
int64_t total_send_bytes;
char *filename;
if (arg == NULL)
{
return EINVAL;
}
filename = (char *)arg;
return tcpsendfile(sock, filename, file_size, \
SF_G_NETWORK_TIMEOUT, &total_send_bytes);
}
int main(int argc, char *argv[])
{
char *conf_filename;
char *local_filename;
ConnectionInfo *pTrackerServer;
ConnectionInfo *pStorageServer;
int result;
ConnectionInfo storageServer;
char group_name[FDFS_GROUP_NAME_MAX_LEN + 1];
char remote_filename[256];
char master_filename[256];
FDFSMetaData meta_list[32];
int meta_count;
int i;
FDFSMetaData *pMetaList;
char formatted_ip[FORMATTED_IP_SIZE];
char token[32 + 1];
char file_id[128];
char file_url[256];
char szDatetime[20];
int url_len;
time_t ts;
char *file_buff;
int64_t file_size;
char *operation;
char *meta_buff;
int store_path_index;
FDFSFileInfo file_info;
printf("This is FastDFS client test program v%d.%d.%d\n" \
"\nCopyright (C) 2008, Happy Fish / YuQing\n" \
"\nFastDFS may be copied only under the terms of the GNU General\n" \
"Public License V3, which may be found in the FastDFS source kit.\n" \
"Please visit the FastDFS Home Page http://www.fastken.com/ \n" \
"for more detail.\n\n", g_fdfs_version.major, g_fdfs_version.minor,
g_fdfs_version.patch);
if (argc < 3)
{
printf("Usage: %s \n" \
"\toperation: upload, download, getmeta, setmeta, " \
"delete and query_servers\n", argv[0]);
return 1;
}
log_init();
//g_log_context.log_level = LOG_DEBUG;
conf_filename = argv[1];
operation = argv[2];
if ((result=fdfs_client_init(conf_filename)) != 0)
{
return result;
}
pTrackerServer = tracker_get_connection();
if (pTrackerServer == NULL)
{
fdfs_client_destroy();
return errno != 0 ? errno : ECONNREFUSED;
}
pStorageServer = NULL;
*group_name = '\0';
local_filename = NULL;
if (strcmp(operation, "upload") == 0)
{
int upload_type;
char *prefix_name;
const char *file_ext_name;
char slave_filename[256];
int slave_filename_len;
if (argc < 4)
{
printf("Usage: %s upload " \
" [FILE | BUFF | CALLBACK] \n",\
argv[0]);
fdfs_client_destroy();
return EINVAL;
}
local_filename = argv[3];
if (argc == 4)
{
upload_type = FDFS_UPLOAD_BY_FILE;
}
else
{
if (strcmp(argv[4], "BUFF") == 0)
{
upload_type = FDFS_UPLOAD_BY_BUFF;
}
else if (strcmp(argv[4], "CALLBACK") == 0)
{
upload_type = FDFS_UPLOAD_BY_CALLBACK;
}
else
{
upload_type = FDFS_UPLOAD_BY_FILE;
}
}
store_path_index = 0;
{
ConnectionInfo storageServers[FDFS_MAX_SERVERS_EACH_GROUP];
ConnectionInfo *pServer;
ConnectionInfo *pServerEnd;
int storage_count;
if ((result=tracker_query_storage_store_list_without_group( \
pTrackerServer, storageServers, \
FDFS_MAX_SERVERS_EACH_GROUP, &storage_count, \
group_name, &store_path_index)) == 0)
{
printf("tracker_query_storage_store_list_without_group: \n");
pServerEnd = storageServers + storage_count;
for (pServer=storageServers; pServerip_addr, pServer->port);
}
printf("\n");
}
}
if ((result=tracker_query_storage_store(pTrackerServer, \
&storageServer, group_name, &store_path_index)) != 0)
{
fdfs_client_destroy();
printf("tracker_query_storage fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
return result;
}
printf("group_name=%s, ip_addr=%s, port=%d\n", \
group_name, storageServer.ip_addr, \
storageServer.port);
if ((pStorageServer=tracker_make_connection(&storageServer, \
&result)) == NULL)
{
fdfs_client_destroy();
return result;
}
memset(&meta_list, 0, sizeof(meta_list));
meta_count = 0;
strcpy(meta_list[meta_count].name, "ext_name");
strcpy(meta_list[meta_count].value, "jpg");
meta_count++;
strcpy(meta_list[meta_count].name, "width");
strcpy(meta_list[meta_count].value, "160");
meta_count++;
strcpy(meta_list[meta_count].name, "height");
strcpy(meta_list[meta_count].value, "80");
meta_count++;
strcpy(meta_list[meta_count].name, "file_size");
strcpy(meta_list[meta_count].value, "115120");
meta_count++;
file_ext_name = fdfs_get_file_ext_name(local_filename);
*group_name = '\0';
if (upload_type == FDFS_UPLOAD_BY_FILE)
{
result = storage_upload_by_filename(pTrackerServer, \
pStorageServer, store_path_index, \
local_filename, file_ext_name, \
meta_list, meta_count, \
group_name, remote_filename);
printf("storage_upload_by_filename\n");
}
else if (upload_type == FDFS_UPLOAD_BY_BUFF)
{
char *file_content;
if ((result=getFileContent(local_filename, \
&file_content, &file_size)) == 0)
{
result = storage_upload_by_filebuff(pTrackerServer, \
pStorageServer, store_path_index, \
file_content, file_size, file_ext_name, \
meta_list, meta_count, \
group_name, remote_filename);
free(file_content);
}
printf("storage_upload_by_filebuff\n");
}
else
{
struct stat stat_buf;
if (stat(local_filename, &stat_buf) == 0 && \
S_ISREG(stat_buf.st_mode))
{
file_size = stat_buf.st_size;
result = storage_upload_by_callback(pTrackerServer, \
pStorageServer, store_path_index, \
uploadFileCallback, local_filename, \
file_size, file_ext_name, \
meta_list, meta_count, \
group_name, remote_filename);
}
printf("storage_upload_by_callback\n");
}
if (result != 0)
{
printf("upload file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
tracker_close_connection_ex(pStorageServer, true);
fdfs_client_destroy();
return result;
}
sprintf(file_id, "%s/%s", group_name, remote_filename);
url_len = sprintf(file_url, "http://%s/%s",
pStorageServer->ip_addr, file_id);
if (g_anti_steal_token)
{
ts = time(NULL);
fdfs_http_gen_token(&g_anti_steal_secret_key, file_id, \
ts, token);
sprintf(file_url + url_len, "?token=%s&ts=%d", \
token, (int)ts);
}
printf("group_name=%s, remote_filename=%s\n", \
group_name, remote_filename);
fdfs_get_file_info(group_name, remote_filename, &file_info);
printf("source ip address: %s\n", file_info.source_ip_addr);
printf("file timestamp=%s\n", formatDatetime(
file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \
szDatetime, sizeof(szDatetime)));
printf("file size=%"PRId64"\n", file_info.file_size);
printf("file crc32=%u\n", file_info.crc32);
printf("example file url: %s\n", file_url);
strcpy(master_filename, remote_filename);
*remote_filename = '\0';
if (upload_type == FDFS_UPLOAD_BY_FILE)
{
prefix_name = "_big";
result = storage_upload_slave_by_filename(pTrackerServer,
NULL, local_filename, master_filename, \
prefix_name, file_ext_name, \
meta_list, meta_count, \
group_name, remote_filename);
printf("storage_upload_slave_by_filename\n");
}
else if (upload_type == FDFS_UPLOAD_BY_BUFF)
{
char *file_content;
prefix_name = "1024x1024";
if ((result=getFileContent(local_filename, \
&file_content, &file_size)) == 0)
{
result = storage_upload_slave_by_filebuff(pTrackerServer, \
NULL, file_content, file_size, master_filename,
prefix_name, file_ext_name, \
meta_list, meta_count, \
group_name, remote_filename);
free(file_content);
}
printf("storage_upload_slave_by_filebuff\n");
}
else
{
struct stat stat_buf;
prefix_name = "-small";
if (stat(local_filename, &stat_buf) == 0 && \
S_ISREG(stat_buf.st_mode))
{
file_size = stat_buf.st_size;
result = storage_upload_slave_by_callback(pTrackerServer, \
NULL, uploadFileCallback, local_filename, \
file_size, master_filename, prefix_name, \
file_ext_name, meta_list, meta_count, \
group_name, remote_filename);
}
printf("storage_upload_slave_by_callback\n");
}
if (result != 0)
{
printf("upload slave file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
tracker_close_connection_ex(pStorageServer, true);
fdfs_client_destroy();
return result;
}
sprintf(file_id, "%s/%s", group_name, remote_filename);
url_len = sprintf(file_url, "http://%s/%s",
pStorageServer->ip_addr, file_id);
if (g_anti_steal_token)
{
ts = time(NULL);
fdfs_http_gen_token(&g_anti_steal_secret_key, file_id, \
ts, token);
sprintf(file_url + url_len, "?token=%s&ts=%d", \
token, (int)ts);
}
printf("group_name=%s, remote_filename=%s\n", \
group_name, remote_filename);
fdfs_get_file_info(group_name, remote_filename, &file_info);
printf("source ip address: %s\n", file_info.source_ip_addr);
printf("file timestamp=%s\n", formatDatetime(
file_info.create_timestamp, "%Y-%m-%d %H:%M:%S", \
szDatetime, sizeof(szDatetime)));
printf("file size=%"PRId64"\n", file_info.file_size);
printf("file crc32=%u\n", file_info.crc32);
printf("example file url: %s\n", file_url);
if (fdfs_gen_slave_filename(master_filename, \
prefix_name, file_ext_name, \
slave_filename, &slave_filename_len) == 0)
{
if (strcmp(remote_filename, slave_filename) != 0)
{
printf("slave_filename=%s\n" \
"remote_filename=%s\n" \
"not equal!\n", \
slave_filename, remote_filename);
}
}
}
else if (strcmp(operation, "download") == 0 ||
strcmp(operation, "getmeta") == 0 ||
strcmp(operation, "setmeta") == 0 ||
strcmp(operation, "query_servers") == 0 ||
strcmp(operation, "delete") == 0)
{
if (argc < 5)
{
printf("Usage: %s %s " \
" \n", \
argv[0], operation);
fdfs_client_destroy();
return EINVAL;
}
snprintf(group_name, sizeof(group_name), "%s", argv[3]);
snprintf(remote_filename, sizeof(remote_filename), \
"%s", argv[4]);
if (strcmp(operation, "setmeta") == 0 ||
strcmp(operation, "delete") == 0)
{
result = tracker_query_storage_update(pTrackerServer, \
&storageServer, group_name, remote_filename);
}
else if (strcmp(operation, "query_servers") == 0)
{
ConnectionInfo storageServers[FDFS_MAX_SERVERS_EACH_GROUP];
int server_count;
result = tracker_query_storage_list(pTrackerServer, \
storageServers, FDFS_MAX_SERVERS_EACH_GROUP, \
&server_count, group_name, remote_filename);
if (result != 0)
{
printf("tracker_query_storage_list fail, "\
"group_name=%s, filename=%s, " \
"error no: %d, error info: %s\n", \
group_name, remote_filename, \
result, STRERROR(result));
}
else
{
printf("server list (%d):\n", server_count);
for (i=0; i= 6)
{
local_filename = argv[5];
if (strcmp(local_filename, "CALLBACK") == 0)
{
FILE *fp;
fp = fopen(local_filename, "wb");
if (fp == NULL)
{
result = errno != 0 ? errno : EPERM;
printf("open file \"%s\" fail, " \
"errno: %d, error info: %s", \
local_filename, result, \
STRERROR(result));
}
else
{
result = storage_download_file_ex( \
pTrackerServer, pStorageServer, \
group_name, remote_filename, 0, 0, \
writeToFileCallback, fp, &file_size);
fclose(fp);
}
}
else
{
result = storage_download_file_to_file( \
pTrackerServer, pStorageServer, \
group_name, remote_filename, \
local_filename, &file_size);
}
}
else
{
file_buff = NULL;
if ((result=storage_download_file_to_buff( \
pTrackerServer, pStorageServer, \
group_name, remote_filename, \
&file_buff, &file_size)) == 0)
{
local_filename = strrchr( \
remote_filename, '/');
if (local_filename != NULL)
{
local_filename++; //skip /
}
else
{
local_filename=remote_filename;
}
result = writeToFile(local_filename, \
file_buff, file_size);
free(file_buff);
}
}
if (result == 0)
{
printf("download file success, " \
"file size=%"PRId64", file save to %s\n", \
file_size, local_filename);
}
else
{
printf("download file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
}
}
else if (strcmp(operation, "getmeta") == 0)
{
if ((result=storage_get_metadata(pTrackerServer, \
pStorageServer, group_name, remote_filename, \
&pMetaList, &meta_count)) == 0)
{
printf("get meta data success, " \
"meta count=%d\n", meta_count);
for (i=0; i %s " \
" " \
" \n" \
"\top_flag: %c for overwrite, " \
"%c for merge\n" \
"\tmetadata_list: name1=value1," \
"name2=value2,...\n", \
argv[0], operation, \
STORAGE_SET_METADATA_FLAG_OVERWRITE, \
STORAGE_SET_METADATA_FLAG_MERGE);
fdfs_client_destroy();
return EINVAL;
}
meta_buff = strdup(argv[6]);
if (meta_buff == NULL)
{
printf("Out of memory!\n");
return ENOMEM;
}
pMetaList = fdfs_split_metadata_ex(meta_buff, \
',', '=', &meta_count, &result);
if (pMetaList == NULL)
{
printf("Out of memory!\n");
free(meta_buff);
return ENOMEM;
}
if ((result=storage_set_metadata(pTrackerServer, \
NULL, group_name, remote_filename, \
pMetaList, meta_count, *argv[5])) == 0)
{
printf("set meta data success\n");
}
else
{
printf("setmeta fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
}
free(meta_buff);
free(pMetaList);
}
else if(strcmp(operation, "delete") == 0)
{
if ((result=storage_delete_file(pTrackerServer, \
NULL, group_name, remote_filename)) == 0)
{
printf("delete file success\n");
}
else
{
printf("delete file fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
}
}
}
else
{
fdfs_client_destroy();
printf("invalid operation: %s\n", operation);
return EINVAL;
}
/* for test only */
if ((result=fdfs_active_test(pTrackerServer)) != 0)
{
format_ip_address(pTrackerServer->ip_addr, formatted_ip);
printf("active_test to tracker server %s:%u fail, errno: %d\n",
formatted_ip, pTrackerServer->port, result);
}
/* for test only */
if ((result=fdfs_active_test(pStorageServer)) != 0)
{
format_ip_address(pStorageServer->ip_addr, formatted_ip);
printf("active_test to storage server %s:%u fail, errno: %d\n",
formatted_ip, pStorageServer->port, result);
}
tracker_close_connection_ex(pStorageServer, true);
tracker_close_connection_ex(pTrackerServer, true);
fdfs_client_destroy();
return result;
}
================================================
FILE: client/fdfs_test1.c
================================================
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include