diff --git a/PYTHON_README.md b/PYTHON_README.md new file mode 100644 index 0000000..8b4480f --- /dev/null +++ b/PYTHON_README.md @@ -0,0 +1,197 @@ +# MongoDB Python Client Guide + +This guide explains how to install the necessary Python packages and use the provided example script to connect to your MongoDB instance. + +## Prerequisites + +- Python 3.6 or higher +- pip (Python package manager) +- Access to your MongoDB instance (running in Docker or elsewhere) + +## Installation Instructions + +1. **Install the PyMongo package** + + PyMongo is the official MongoDB driver for Python. Install it using pip: + + ```bash + pip install pymongo + ``` + + If you're using a virtual environment (recommended): + + ```bash + # Create a virtual environment + python -m venv venv + + # Activate the virtual environment + # On Linux/macOS: + source venv/bin/activate + # On Windows: + venv\Scripts\activate + + # Install PyMongo + pip install pymongo + ``` + +2. **Configure environment variables (optional)** + + The example script uses environment variables for configuration. You can set them in your shell: + + ```bash + export MONGO_HOST="your-mongodb-host" + export MONGO_PORT="27017" + export MONGO_USERNAME="admin" + export MONGO_PASSWORD="your-password" + export MONGO_AUTH_DB="admin" + ``` + + Alternatively, you can modify the default values directly in the script. + +## Using the Example Script + +The `python_example.py` script demonstrates basic MongoDB operations: + +1. Connecting to MongoDB with authentication +2. Listing available databases +3. Creating a sample database and collection +4. Inserting sample data +5. Querying the data + +### Running the Script + +```bash +python python_example.py +``` + +### Expected Output + +``` +šŸ”„ Connecting to MongoDB... +āœ… Successfully connected to MongoDB at localhost:27017 + +šŸ“š Available databases: + - admin + - config + - local + +šŸ”„ Creating sample data... +āœ… Successfully inserted 3 documents into sample_db.users +šŸ“Š Total documents in users: 3 + +šŸ”„ Querying data... +šŸ” All users: + - John Doe (john.doe@example.com), Age: 30 + - Jane Smith (jane.smith@example.com), Age: 25 + - Bob Johnson (bob.johnson@example.com), Age: 35 + +šŸ” Users older than 30: + - Bob Johnson (bob.johnson@example.com), Age: 35 + +šŸ‘‹ Connection closed +``` + +## Modifying the Script for Your Needs + +### Connecting to Your MongoDB Instance + +Update the connection parameters at the top of the script: + +```python +MONGO_HOST = "your-mongodb-host" # Replace with your MongoDB host IP +MONGO_PORT = 27017 +MONGO_USERNAME = "admin" # Replace with your MongoDB username +MONGO_PASSWORD = "your-password" # Replace with your MongoDB password +MONGO_AUTH_DB = "admin" +``` + +### Creating Your Own Collections and Queries + +The example script provides a template that you can modify: + +1. To create a different collection, modify the `create_sample_data` function +2. To perform different queries, modify the `query_data` function + +## Common MongoDB Operations in Python + +### Insert a Document + +```python +result = collection.insert_one({"name": "Alice", "age": 29}) +print(f"Inserted document ID: {result.inserted_id}") +``` + +### Find Documents + +```python +# Find one document +user = collection.find_one({"name": "John Doe"}) +print(user) + +# Find documents with a filter +users = collection.find({"age": {"$gt": 25}}) +for user in users: + print(user) +``` + +### Update Documents + +```python +# Update one document +result = collection.update_one( + {"name": "John Doe"}, + {"$set": {"age": 31}} +) +print(f"Modified {result.modified_count} document(s)") + +# Update multiple documents +result = collection.update_many( + {"age": {"$lt": 30}}, + {"$inc": {"age": 1}} +) +print(f"Modified {result.modified_count} document(s)") +``` + +### Delete Documents + +```python +# Delete one document +result = collection.delete_one({"name": "John Doe"}) +print(f"Deleted {result.deleted_count} document(s)") + +# Delete multiple documents +result = collection.delete_many({"age": {"$lt": 25}}) +print(f"Deleted {result.deleted_count} document(s)") +``` + +## Troubleshooting + +### Connection Issues + +If you can't connect to MongoDB: + +1. Verify that your MongoDB container is running: + ```bash + docker-compose ps + ``` + +2. Check if you can reach MongoDB from your host: + ```bash + telnet your-mongodb-host 27017 + ``` + +3. Ensure your MongoDB credentials are correct + +### Authentication Errors + +If you see authentication errors: + +1. Verify your username and password +2. Make sure you're using the correct authentication database (usually "admin") +3. Check if MongoDB is configured with authentication enabled + +## Further Resources + +- [PyMongo Documentation](https://pymongo.readthedocs.io/) +- [MongoDB Python Driver Tutorials](https://www.mongodb.com/docs/drivers/python/) +- [MongoDB Query Operators](https://www.mongodb.com/docs/manual/reference/operator/query/) diff --git a/load_test.py b/load_test.py new file mode 100644 index 0000000..d601f56 --- /dev/null +++ b/load_test.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +""" +MongoDB Load Test Script + +This script tests MongoDB performance with multiple concurrent connections +by running the example script in multiple threads. +""" + +import time +import threading +import concurrent.futures +from datetime import datetime +import sys +import os + +# Import the main function from our example script +from python_example import main as run_mongodb_operations + +# Configuration +NUM_THREADS = 100 # Number of concurrent connections +TIMEOUT = 120 # Maximum time to wait for all threads (seconds) + + +def run_thread(thread_id): + """ + Function to run in each thread. + + Args: + thread_id: The thread identifier + + Returns: + Tuple of (thread_id, success, elapsed_time) + """ + start_time = time.time() + try: + # Run the MongoDB operations + success = run_mongodb_operations(thread_id) + elapsed = time.time() - start_time + return (thread_id, success, elapsed) + except Exception as e: + elapsed = time.time() - start_time + print(f"[Thread {thread_id}] āŒ Error: {e}") + return (thread_id, False, elapsed) + + +def main(): + """Main function to run the load test.""" + print(f"šŸš€ Starting MongoDB load test with {NUM_THREADS} concurrent connections") + print(f"ā±ļø Timeout set to {TIMEOUT} seconds") + print("-" * 80) + + start_time = time.time() + + # Results tracking + successful_threads = 0 + failed_threads = 0 + times = [] + + # Use a thread pool to manage the concurrent connections + with concurrent.futures.ThreadPoolExecutor(max_workers=NUM_THREADS) as executor: + # Submit all threads + future_to_thread = { + executor.submit(run_thread, i): i for i in range(NUM_THREADS) + } + + # Process results as they complete + for future in concurrent.futures.as_completed(future_to_thread, timeout=TIMEOUT): + thread_id = future_to_thread[future] + try: + thread_id, success, elapsed = future.result() + times.append(elapsed) + if success: + successful_threads += 1 + else: + failed_threads += 1 + except Exception as e: + print(f"[Thread {thread_id}] āŒ Generated an exception: {e}") + failed_threads += 1 + + # Calculate statistics + total_time = time.time() - start_time + avg_time = sum(times) / len(times) if times else 0 + min_time = min(times) if times else 0 + max_time = max(times) if times else 0 + + # Print results + print("\n" + "=" * 80) + print(f"šŸ“Š MongoDB Load Test Results:") + print(f" Total threads: {NUM_THREADS}") + print(f" Successful threads: {successful_threads}") + print(f" Failed threads: {failed_threads}") + print(f" Total test time: {total_time:.2f} seconds") + print(f" Average thread time: {avg_time:.2f} seconds") + print(f" Minimum thread time: {min_time:.2f} seconds") + print(f" Maximum thread time: {max_time:.2f} seconds") + print(f" Throughput: {successful_threads / total_time:.2f} operations/second") + print("=" * 80) + + # Return exit code based on success + return 0 if failed_threads == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/mongodb_load_tester.py b/mongodb_load_tester.py new file mode 100644 index 0000000..e9b9f9e --- /dev/null +++ b/mongodb_load_tester.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +""" +MongoDB Load Test - All-in-One Script + +This script combines MongoDB operations and load testing in a single file. +It creates multiple threads to test MongoDB's performance under concurrent load. +""" + +import os +import sys +import time +import threading +import concurrent.futures +from datetime import datetime +from pymongo import MongoClient +from pymongo.errors import ConnectionFailure, OperationFailure + +# ===== Configuration ===== +# MongoDB connection details +MONGO_HOST = os.environ.get("MONGO_HOST", "localhost") +MONGO_PORT = int(os.environ.get("MONGO_PORT", "27017")) +MONGO_USERNAME = os.environ.get("MONGO_USERNAME", "admin") +MONGO_PASSWORD = os.environ.get("MONGO_PASSWORD", "password") +MONGO_AUTH_DB = os.environ.get("MONGO_AUTH_DB", "admin") + +# Load test settings +NUM_THREADS = 100 # Number of concurrent connections +TIMEOUT = 120 # Maximum time to wait for all threads (seconds) + +# ===== MongoDB Operations ===== +def connect_to_mongodb(): + """Connect to MongoDB and return the client object.""" + try: + # Create a MongoDB client with authentication + connection_string = f"mongodb://{MONGO_USERNAME}:{MONGO_PASSWORD}@{MONGO_HOST}:{MONGO_PORT}/?authSource={MONGO_AUTH_DB}" + client = MongoClient(connection_string) + + # Check if the connection is successful + client.admin.command("ping") + return client + except ConnectionFailure as e: + print(f"āŒ Failed to connect to MongoDB: {e}") + raise + except OperationFailure as e: + print(f"āŒ Authentication failed: {e}") + raise + + +def list_databases(client, thread_id=None): + """List all databases in the MongoDB instance.""" + prefix = f"[Thread {thread_id}] " if thread_id is not None else "" + try: + databases = client.list_database_names() + if thread_id is None: # Only print for main thread or when no threading + print(f"\n{prefix}šŸ“š Available databases:") + for db in databases: + print(f" - {db}") + return databases + except Exception as e: + print(f"{prefix}āŒ Error listing databases: {e}") + return [] + + +def create_sample_data(client, database_name="sample_db", collection_name="users", thread_id=None): + """Create a sample database and collection with some data.""" + prefix = f"[Thread {thread_id}] " if thread_id is not None else "" + try: + db = client[database_name] + collection = db[collection_name] + + # Sample user data + users = [ + { + "name": f"John Doe {thread_id or ''}", + "email": f"john.doe{thread_id or ''}@example.com", + "age": 30, + "created_at": datetime.now(), + }, + { + "name": f"Jane Smith {thread_id or ''}", + "email": f"jane.smith{thread_id or ''}@example.com", + "age": 25, + "created_at": datetime.now(), + }, + { + "name": f"Bob Johnson {thread_id or ''}", + "email": f"bob.johnson{thread_id or ''}@example.com", + "age": 35, + "created_at": datetime.now(), + }, + ] + + # Insert the users + result = collection.insert_many(users) + print( + f"{prefix}āœ… Inserted {len(result.inserted_ids)} documents into {database_name}.{collection_name}" + ) + + # Count documents in the collection + count = collection.count_documents({}) + print(f"{prefix}šŸ“Š Total documents in {collection_name}: {count}") + + return True + except Exception as e: + print(f"{prefix}āŒ Error creating sample data: {e}") + return False + + +def query_data(client, database_name="sample_db", collection_name="users", thread_id=None): + """Query and display data from the collection.""" + prefix = f"[Thread {thread_id}] " if thread_id is not None else "" + try: + db = client[database_name] + collection = db[collection_name] + + # Find all users + if thread_id is None: # Only print details for main thread + print(f"{prefix}šŸ” All users:") + for user in collection.find(): + print(f" - {user['name']} ({user['email']}), Age: {user['age']}") + + # Find users older than 30 + print(f"{prefix}šŸ” Users older than 30:") + for user in collection.find({"age": {"$gt": 30}}): + print(f" - {user['name']} ({user['email']}), Age: {user['age']}") + else: + # For threads, just count to reduce output noise + all_users = collection.count_documents({}) + older_users = collection.count_documents({"age": {"$gt": 30}}) + print(f"{prefix}šŸ” Found {all_users} users, {older_users} older than 30") + + return True + except Exception as e: + print(f"{prefix}āŒ Error querying data: {e}") + return False + + +def run_mongodb_operations(thread_id=None): + """Run a complete set of MongoDB operations.""" + prefix = f"[Thread {thread_id}] " if thread_id is not None else "" + try: + print(f"{prefix}šŸ”„ Connecting to MongoDB...") + client = connect_to_mongodb() + + # List databases (only in main thread to reduce output) + if thread_id is None or thread_id == 0: + list_databases(client, thread_id) + + # Create sample data with unique collection name if threading + collection_suffix = f"_thread_{thread_id}" if thread_id is not None else "" + create_sample_data( + client, + database_name="sample_db", + collection_name=f"users{collection_suffix}", + thread_id=thread_id + ) + + # Query data + query_data( + client, + database_name="sample_db", + collection_name=f"users{collection_suffix}", + thread_id=thread_id + ) + + # Close the connection + client.close() + print(f"{prefix}šŸ‘‹ Connection closed") + return True + except Exception as e: + print(f"{prefix}āŒ Error in MongoDB operations: {e}") + return False + + +# ===== Load Testing ===== +def run_thread(thread_id): + """ + Function to run in each thread. + + Args: + thread_id: The thread identifier + + Returns: + Tuple of (thread_id, success, elapsed_time) + """ + start_time = time.time() + try: + # Run the MongoDB operations + success = run_mongodb_operations(thread_id) + elapsed = time.time() - start_time + return (thread_id, success, elapsed) + except Exception as e: + elapsed = time.time() - start_time + print(f"[Thread {thread_id}] āŒ Error: {e}") + return (thread_id, False, elapsed) + + +def run_load_test(num_threads=NUM_THREADS, timeout=TIMEOUT): + """Run the load test with multiple threads.""" + print(f"šŸš€ Starting MongoDB load test with {num_threads} concurrent connections") + print(f"ā±ļø Timeout set to {timeout} seconds") + print("-" * 80) + + start_time = time.time() + + # Results tracking + successful_threads = 0 + failed_threads = 0 + times = [] + + # Use a thread pool to manage the concurrent connections + with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor: + # Submit all threads + future_to_thread = { + executor.submit(run_thread, i): i for i in range(num_threads) + } + + # Process results as they complete + for future in concurrent.futures.as_completed(future_to_thread, timeout=timeout): + thread_id = future_to_thread[future] + try: + thread_id, success, elapsed = future.result() + times.append(elapsed) + if success: + successful_threads += 1 + else: + failed_threads += 1 + except Exception as e: + print(f"[Thread {thread_id}] āŒ Generated an exception: {e}") + failed_threads += 1 + + # Calculate statistics + total_time = time.time() - start_time + avg_time = sum(times) / len(times) if times else 0 + min_time = min(times) if times else 0 + max_time = max(times) if times else 0 + + # Print results + print("\n" + "=" * 80) + print(f"šŸ“Š MongoDB Load Test Results:") + print(f" Total threads: {num_threads}") + print(f" Successful threads: {successful_threads}") + print(f" Failed threads: {failed_threads}") + print(f" Total test time: {total_time:.2f} seconds") + print(f" Average thread time: {avg_time:.2f} seconds") + print(f" Minimum thread time: {min_time:.2f} seconds") + print(f" Maximum thread time: {max_time:.2f} seconds") + print(f" Throughput: {successful_threads / total_time:.2f} operations/second") + print("=" * 80) + + # Return exit code based on success + return 0 if failed_threads == 0 else 1 + + +def main(): + """Main function to run either a single test or load test.""" + if len(sys.argv) > 1 and sys.argv[1] == "--single": + # Run a single MongoDB operation test + print("šŸ”„ Running single MongoDB test...") + success = run_mongodb_operations() + return 0 if success else 1 + else: + # Run the load test + return run_load_test() + + +if __name__ == "__main__": + sys.exit(main()) + + + +""" +================================================================================ +šŸ“Š MongoDB Load Test Results: + Total threads: 100 + Successful threads: 100 + Failed threads: 0 + Total test time: 0.99 seconds + Average thread time: 0.62 seconds + Minimum thread time: 0.13 seconds + Maximum thread time: 0.74 seconds + Throughput: 100.71 operations/second +================================================================================ Enough for a simple mongo server that handles maximum connection of 100 +""" diff --git a/python_example.py b/python_example.py new file mode 100644 index 0000000..740a481 --- /dev/null +++ b/python_example.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +""" +Example Python script to connect to MongoDB and perform basic operations. +""" + +import os +import sys +from datetime import datetime +from pymongo import MongoClient +from pymongo.errors import ConnectionFailure, OperationFailure + +# MongoDB connection details +MONGO_HOST = os.environ.get("MONGO_HOST", "localhost") # Replace with your MongoDB host +MONGO_PORT = int(os.environ.get("MONGO_PORT", "27017")) +MONGO_USERNAME = os.environ.get( + "MONGO_USERNAME", "admin" +) # Replace with your MongoDB username +MONGO_PASSWORD = os.environ.get( + "MONGO_PASSWORD", "password" +) # Replace with your MongoDB password +MONGO_AUTH_DB = os.environ.get("MONGO_AUTH_DB", "admin") + + +def connect_to_mongodb(): + """Connect to MongoDB and return the client object.""" + try: + # Create a MongoDB client with authentication + connection_string = f"mongodb://{MONGO_USERNAME}:{MONGO_PASSWORD}@{MONGO_HOST}:{MONGO_PORT}/?authSource={MONGO_AUTH_DB}" + client = MongoClient(connection_string) + + # Check if the connection is successful + client.admin.command("ping") + print(f"āœ… Successfully connected to MongoDB at {MONGO_HOST}:{MONGO_PORT}") + return client + except ConnectionFailure as e: + print(f"āŒ Failed to connect to MongoDB: {e}") + sys.exit(1) + except OperationFailure as e: + print(f"āŒ Authentication failed: {e}") + sys.exit(1) + + +def list_databases(client): + """List all databases in the MongoDB instance.""" + try: + databases = client.list_database_names() + print("\nšŸ“š Available databases:") + for db in databases: + print(f" - {db}") + return databases + except Exception as e: + print(f"āŒ Error listing databases: {e}") + return [] + + +def create_sample_data(client, database_name="sample_db", collection_name="users"): + """Create a sample database and collection with some data.""" + try: + db = client[database_name] + collection = db[collection_name] + + # Sample user data + users = [ + { + "name": "John Doe", + "email": "john.doe@example.com", + "age": 30, + "created_at": datetime.now(), + }, + { + "name": "Jane Smith", + "email": "jane.smith@example.com", + "age": 25, + "created_at": datetime.now(), + }, + { + "name": "Bob Johnson", + "email": "bob.johnson@example.com", + "age": 35, + "created_at": datetime.now(), + }, + ] + + # Insert the users + result = collection.insert_many(users) + print( + f"\nāœ… Successfully inserted {len(result.inserted_ids)} documents into {database_name}.{collection_name}" + ) + + # Count documents in the collection + count = collection.count_documents({}) + print(f"šŸ“Š Total documents in {collection_name}: {count}") + + return True + except Exception as e: + print(f"āŒ Error creating sample data: {e}") + return False + + +def query_data(client, database_name="sample_db", collection_name="users"): + """Query and display data from the collection.""" + try: + db = client[database_name] + collection = db[collection_name] + + # Find all users + print("\nšŸ” All users:") + for user in collection.find(): + print(f" - {user['name']} ({user['email']}), Age: {user['age']}") + + # Find users older than 30 + print("\nšŸ” Users older than 30:") + for user in collection.find({"age": {"$gt": 30}}): + print(f" - {user['name']} ({user['email']}), Age: {user['age']}") + + return True + except Exception as e: + print(f"āŒ Error querying data: {e}") + return False + + +def main(thread_id=None): + """Main function to demonstrate MongoDB operations. + + Args: + thread_id: Optional thread identifier for concurrent testing + """ + prefix = f"[Thread {thread_id}] " if thread_id is not None else "" + print(f"{prefix}šŸ”„ Connecting to MongoDB...") + client = connect_to_mongodb() + + # List databases + list_databases(client) + + # Create sample data with unique collection name if threading + collection_suffix = f"_thread_{thread_id}" if thread_id is not None else "" + print(f"\n{prefix}šŸ”„ Creating sample data...") + create_sample_data(client, collection_name=f"users{collection_suffix}") + + # Query data + print(f"\n{prefix}šŸ”„ Querying data...") + query_data(client, collection_name=f"users{collection_suffix}") + + # Close the connection + client.close() + print(f"\n{prefix}šŸ‘‹ Connection closed") + + return True + + +if __name__ == "__main__": + main()